Warn on url values new URL() accepts but are not real URLs; default to http/https

.bumpy/strict-url-validation.md

@@ -0,0 +1,6 @@
+---
+varlock: minor
+env-spec-language: minor
+---
+
+The `url` type now warns on values that only parse because `new URL()` is permissive: anything not written as `scheme://...` (like an SSH key fingerprint `SHA256:...` or `user:pass`), whitespace or control characters, a missing host, and malformed `http`/`https` hosts the parser would otherwise rewrite (`https:///example.com`, `http://0x7f.1`, `https://a..b`). It also warns on protocols other than `http`/`https` when `allowedProtocols` is not set. These become errors in the next major version. New options: `allowedProtocols="*"` allows any protocol, and `allowNoHost=true` allows host-less URLs like `sqlite:///app.db`.

packages/varlock-website/src/content/docs/guides/mcp.mdx

@@ -70,7 +70,7 @@ Use varlock's validation features to ensure data integrity:
 # @type=string(startsWith="sk_", minLength=20)
 API_KEY=
 
-# @type=url
+# @type=url(allowedProtocols=[postgres, postgresql])
 DATABASE_URL=
 ```
 

packages/varlock-website/src/content/docs/guides/mcp/local.mdx

@@ -21,7 +21,7 @@ Create a `.env.schema` file for your MCP server:
 # ---
 
 # Database connection for MCP server
-# @type=url
+# @type=url(allowedProtocols=[postgres, postgresql])
 DATABASE_URL=
 
 # API key for external service

packages/varlock-website/src/content/docs/guides/mcp/remote.mdx

@@ -87,7 +87,7 @@ For production, create environment-specific schema files. See the [environments
 APP_ENV=development
 
 # Database connection
-# @type=url
+# @type=url(allowedProtocols=[postgres, postgresql])
 DATABASE_URL=
 
 # External API credentials

packages/varlock-website/src/content/docs/guides/monorepos.mdx

@@ -57,7 +57,7 @@ If you would rather keep a single `.env.schema` at the repo root than maintain a
 ```env-spec title=".env.schema (repo root)"
 # @tag(web) @type=url
 NEXT_PUBLIC_SERVER_URL=
-# @tag(api) @type=url
+# @tag(api) @type=url(allowedProtocols=[postgres, postgresql])
 DATABASE_URL=
 # @tag(web, api)
 SENTRY_DSN=

packages/varlock-website/src/content/docs/reference/data-types.mdx

@@ -163,9 +163,22 @@ MY_BOOL=true
 
 <div>
 ### `url`
+Checks for an absolute URL written as `scheme://...`, using `http` or `https` unless `allowedProtocols` says otherwise.
+
+`new URL()` alone accepts any `word:rest` string (so an SSH key fingerprint like `SHA256:nThbg6...` parses as a URL with a `sha256:` scheme) and silently repairs malformed input, so varlock also flags a value that:
+- does not start with `scheme://` (`mailto:` and `data:` URLs are not accepted)
+- contains whitespace or control characters anywhere, including leading or trailing
+- has no host, unless `allowNoHost=true`. For `http`, `https`, `ws`, `wss`, and `ftp` a host is always required (`https:///example.com` is flagged)
+- for those same schemes, uses a backslash, a percent-encoded host, IPv4 shorthand like `http://0x7f.1` (write `http://127.0.0.1`), or a hostname with empty labels or labels starting or ending in `-`
+
+:::caution
+These checks, and the `http`/`https` default for `allowedProtocols`, are warnings for now and become errors in the next major version. To silence a protocol warning, set `allowedProtocols` explicitly.
+:::
+
 **Options:**
 - `prependHttps` (boolean): Automatically prepend "https://" if no protocol is specified. A `host:port` value like `localhost:3000` counts as having no protocol, unless `allowedProtocols` lists the part before the colon
-- `allowedProtocols` (string[]): List of allowed protocols. Protocol names are case-insensitive and can include the trailing colon. If omitted, any valid URL protocol is allowed
+- `allowedProtocols` (string[]): List of allowed protocols. Protocol names are case-insensitive and can include the trailing colon. If omitted, only `http` and `https` are allowed (a warning until the next major version). Use `allowedProtocols="*"` to allow any protocol
+- `allowNoHost` (boolean): Allow URLs with no host, like `sqlite:///app.db`, `file:///etc/hosts`, or `postgresql:///mydb?host=/var/run/postgresql`. Has no effect for `http`, `https`, `ws`, `wss`, and `ftp`, which always need a host
 - `allowedDomains` (string[]): List of allowed hosts, matched in full and case-insensitively. An entry without a port allows any port, so `[localhost]` accepts `http://localhost:3000`; add a port to pin it (`["localhost:3000"]`). Each entry must be a hostname with an optional port: a scheme, path, or credentials in an entry is an error, as is an empty list. A bare string is treated as a single host; list two or more as an array. When set, the URL's host must be written plainly so every URL parser reads the same host: values with backslashes, whitespace, a percent-encoded host, a non-ASCII host (write internationalized domains in punycode, like `xn--mnchen-3ya.de`), IPv4 shorthand (`http://0x7f.1`), or no `scheme://host` prefix are rejected
 - `noTrailingSlash` (boolean): Disallow a trailing slash on the URL, so the value is safe to concatenate onto. A root `/` counts, so `https://example.com/` is rejected while `https://example.com` passes
 - `matches` (RegExp): Regular expression the full URL must match, as a [`regex()`](/reference/functions#regex) call

packages/varlock/README.md

@@ -27,7 +27,7 @@ API_PORT=
 
 # API url including expansion of another env var
 # @required @type=url
-API_URL=localhost:${API_PORT}
+API_URL=http://localhost:${API_PORT}
 
 # API key with validation, securely fetched from 1Password
 # @required @sensitive @type=string(startsWith=sk-)

packages/varlock/src/env-graph/lib/config-item.ts

@@ -1200,7 +1200,14 @@ export class ConfigItem {
       if (
         validateResult instanceof Error
         || (_.isArray(validateResult) && validateResult[0] instanceof Error)
-      ) throw validateResult;
+      ) {
+        const resultErrors: Array<Error> = _.isArray(validateResult) ? validateResult : [validateResult];
+        // warnings alone are advisory, so keep them and carry on with the checks below
+        // (the sensitive-value checks must still run); anything else stops validation here
+        const onlyWarnings = resultErrors.every((e) => e instanceof ValidationError && e.isWarning);
+        if (!onlyWarnings) throw validateResult;
+        this.validationErrors = resultErrors as Array<ValidationError>;
+      }
       // validation result is supposed to be `true` or error(s), but we'll check for `false` just in case
       if ((validateResult as any) === false) {
         throw new ValidationError('validation failed with `false` return value');

packages/varlock/src/env-graph/lib/data-types.ts

@@ -1,7 +1,7 @@
 import { createHash } from 'node:crypto';
 import _ from '@env-spec/utils/my-dash';
 import { type FallbackIfUnknown } from '@env-spec/utils/type-utils';
-import { CoercionError, ValidationError } from './errors';
+import { CoercionError, ValidationError, type ErrorSeverity } from './errors';
 import { parseRegexLikeString } from './regex-like-string';
 import {
   parseDuration, convertDurationFromMs, type DurationUnit,
@@ -440,12 +440,82 @@ function hasUrlScheme(val: string, allowedProtocols?: unknown) {
   ));
 }
 
+// schemes the WHATWG parser treats as "special": it requires a host for these and silently
+// repairs malformed input (`http:foo`, `https:///x`, `\` as `/`, IPv4 shorthand)
+const SPECIAL_URL_SCHEMES = ['http', 'https', 'ws', 'wss', 'ftp'];
+const URL_SCHEME_WITH_AUTHORITY_REGEX = /^([a-z][a-z\d+.-]*):\/\//i;
+const DOTTED_QUAD_IPV4_REGEX = /^\d{1,3}(?:\.\d{1,3}){3}$/;
+/** protocols a `url` accepts when `allowedProtocols` is not set (enforced from the next major) */
+const DEFAULT_URL_PROTOCOLS = ['http', 'https'];
+// The strict url checks below are warnings for now so existing schemas get a release to
+// adapt. TODO(next major): flip to 'error'.
+const STRICT_URL_SEVERITY: ErrorSeverity = 'warning';
+const STRICT_URL_TIP = 'This will be an error in the next major version of varlock.';
+
+/**
+ * Structural checks `new URL()` does not make. On its own it accepts anything shaped like
+ * `word:rest` as a URL with an opaque path, so `SHA256:nThbg6...` (an SSH key fingerprint),
+ * `user:pass`, and `C:\path` all parse, and for special schemes it rewrites malformed input
+ * into something valid rather than rejecting it.
+ *
+ * Returns an error message, or undefined if the URL is acceptable.
+ * Keep in step with `checkUrlStructure` in packages/vscode-plugin/src/diagnostics-core.ts
+ */
+function checkUrlStructure(val: string, url: URL, opts: { allowNoHost?: boolean }): string | undefined {
+  // the parser strips leading/trailing whitespace and removes tabs and newlines anywhere,
+  // so the value would validate while the raw string (whitespace included) is what gets used
+  // eslint-disable-next-line no-control-regex -- matching control characters is the point
+  if (/[\s\x00-\x1f\x7f]/.test(val)) return 'URL must not contain whitespace or control characters';
+
+  const schemeMatch = val.match(URL_SCHEME_WITH_AUTHORITY_REGEX);
+  if (!schemeMatch) return 'URL must start with a scheme followed by `://` (for example `https://`)';
+  const scheme = schemeMatch[1].toLowerCase();
+  const isSpecial = SPECIAL_URL_SCHEMES.includes(scheme);
+
+  const rest = val.slice(schemeMatch[0].length);
+  if (!rest) return 'URL is missing a host';
+  // WHATWG treats `\` as `/` for special schemes, so `https://a.com\@b.com` has host `a.com`
+  if (isSpecial && rest.includes('\\')) return 'URL must not contain backslashes';
+
+  // the host as written, without userinfo or port
+  const authority = rest.split(/[/?#]/)[0];
+  const hostAndPort = authority.slice(authority.lastIndexOf('@') + 1);
+  if (hostAndPort.startsWith('[')) return undefined; // IPv6 literal, already checked by the parser
+  const host = hostAndPort.replace(/:\d*$/, '').toLowerCase();
+
+  if (!host) {
+    // for special schemes an empty written host means the parser took it from the path:
+    // `https:///example.com` has host `example.com`
+    if (isSpecial) return 'URL is missing a host';
+    // other schemes have real host-less forms (`sqlite:///app.db`, `file:///etc/hosts`,
+    // `postgresql:///db?host=/var/run/postgresql`), but they are rare enough to opt into
+    return opts.allowNoHost ? undefined : 'URL is missing a host (set `allowNoHost=true` to allow forms like `sqlite:///app.db`)';
+  }
+  if (!isSpecial) return undefined; // non-special hosts are opaque (may be percent-encoded paths)
+
+  // percent-escapes in the host are decoded by the parser, so `%65xample.com` is example.com
+  if (host.includes('%')) return 'URL host must not contain percent-encoded characters';
+  // IPv4 shorthand is normalized: `http://1` is 0.0.0.1, `http://0x7f.1` is 127.0.0.1
+  if (DOTTED_QUAD_IPV4_REGEX.test(url.hostname) && host !== url.hostname) {
+    return `URL host \`${host}\` must be written as a full IPv4 address (\`${url.hostname}\`)`;
+  }
+  // a single trailing dot is a valid fully-qualified name, any other empty label is not
+  const labels = host.replace(/\.$/, '').split('.');
+  if (labels.some((label) => !label || label.startsWith('-') || label.endsWith('-'))) {
+    return `URL host \`${host}\` is not a valid hostname`;
+  }
+  return undefined;
+}
+
 const UrlDataType = createEnvGraphDataType(
   (settings?: {
     prependHttps?: boolean
     /** Allowed hosts. A bare string is treated as a single host. */
     allowedDomains?: Array<string> | string
-    allowedProtocols?: Array<string>
+    /** Allowed protocols, `http` and `https` if not set. `"*"` (or an entry of `*`) allows any. */
+    allowedProtocols?: Array<string> | string
+    /** Allow URLs with no host, like `sqlite:///app.db` or `file:///etc/hosts`. */
+    allowNoHost?: boolean
     /** Disallow a trailing slash on the URL path. */
     noTrailingSlash?: boolean
     /** A regular expression or string pattern that the full URL must match. */
@@ -468,20 +538,37 @@ const UrlDataType = createEnvGraphDataType(
         throw new ValidationError('Invalid URL');
       }
       const errors = [] as Array<ValidationError>;
-      if (settings?.allowedProtocols) {
-        if (
-          !Array.isArray(settings.allowedProtocols)
-          || settings.allowedProtocols.some((allowedProtocol) => !_.isString(allowedProtocol))
-        ) {
-          errors.push(new ValidationError('allowedProtocols must be an array of strings'));
-        } else {
-          const protocol = url.protocol.replace(/:$/, '').toLowerCase();
-          const allowedProtocols = settings.allowedProtocols.map((allowedProtocol) => (
-            allowedProtocol.replace(/:$/, '').toLowerCase()
+      const ambiguousHostError = settings?.allowedDomains ? findAmbiguousUrlHost(val, url) : undefined;
+      const structureIssue = checkUrlStructure(val, url, { allowNoHost: settings?.allowNoHost });
+      // the allowedDomains check already reports these as an error, so skip the duplicate
+      if (structureIssue && !ambiguousHostError) {
+        errors.push(new ValidationError(structureIssue, { severity: STRICT_URL_SEVERITY, tip: STRICT_URL_TIP }));
+      }
+      const protocol = url.protocol.replace(/:$/, '').toLowerCase();
+      const rawAllowedProtocols = settings?.allowedProtocols;
+      if (rawAllowedProtocols === undefined) {
+        if (!DEFAULT_URL_PROTOCOLS.includes(protocol)) {
+          errors.push(new ValidationError(
+            `Protocol (${protocol}) will not be allowed by default in the next major version of varlock`,
+            {
+              severity: STRICT_URL_SEVERITY,
+              tip: `Only http and https are allowed unless set explicitly: add \`allowedProtocols=[${protocol}]\`, or \`allowedProtocols="*"\` to allow any.`,
+            },
           ));
-          if (!allowedProtocols.includes(protocol)) {
-            errors.push(new ValidationError(`Protocol (${protocol}) is not in allowed list: ${settings.allowedProtocols.join(',')}`));
-          }
+        }
+      } else if (rawAllowedProtocols === '*') {
+        // any protocol
+      } else if (
+        !Array.isArray(rawAllowedProtocols)
+        || rawAllowedProtocols.some((allowedProtocol) => !_.isString(allowedProtocol))
+      ) {
+        errors.push(new ValidationError('allowedProtocols must be an array of strings'));
+      } else {
+        const allowedProtocols = rawAllowedProtocols.map((allowedProtocol) => (
+          allowedProtocol.replace(/:$/, '').toLowerCase()
+        ));
+        if (!allowedProtocols.includes('*') && !allowedProtocols.includes(protocol)) {
+          errors.push(new ValidationError(`Protocol (${protocol}) is not in allowed list: ${rawAllowedProtocols.join(',')}`));
         }
       }
       if (settings?.allowedDomains) {
@@ -525,7 +612,6 @@ const UrlDataType = createEnvGraphDataType(
           // special schemes, so `trusted.example\path` would otherwise parse as a host plus path
           const HOST_ENTRY_REGEX = /^(?:\[[0-9a-f:.]+\]|[^\s\\/?#@:[\]]+)(?::\d+)?$/i;
           const malformedDomains = normalizedDomains.filter((d) => !HOST_ENTRY_REGEX.test(d));
-          const ambiguousHostError = findAmbiguousUrlHost(val, url);
           if (!normalizedDomains.length) {
             // an empty list can never match, so say so rather than rejecting every URL
             // with an allowlist that has nothing in it to report
@@ -1026,7 +1112,8 @@ async function runNestedValidate(type: EnvGraphDataType, val: any): Promise<Arra
 
 /** re-wrap a nested validation error with a positional prefix (array index / object key) */
 function prefixValidationError(prefix: string, err: ValidationError): ValidationError {
-  return new ValidationError(`${prefix} ${err.message}`, { err, tip: err.tip });
+  // keep the severity, so a nested warning (e.g. from `array(url)`) does not become an error
+  return new ValidationError(`${prefix} ${err.message}`, { err, tip: err.tip, severity: err.severity });
 }
 
 const DEFAULT_ARRAY_SEPARATOR = ',';