diff --git a/.gitignore b/.gitignore index bf4f33564ad..59a87d33d04 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ perl/Makefile.config /src/libexpr/parser-tab.output /src/libexpr/nix.tbl /src/libexpr/tests/libnixexpr-tests +/src/libexpr/*.inc.hh # /src/libstore/ *.gen.* diff --git a/flake.nix b/flake.nix index e3b9477a582..a6cb5bef0f7 100644 --- a/flake.nix +++ b/flake.nix @@ -175,6 +175,7 @@ buildPackages.autoconf-archive buildPackages.autoreconfHook buildPackages.pkg-config + buildPackages.nixVersions.nix_2_14 # Tests buildPackages.git diff --git a/src/libexpr/diagnostic.hh b/src/libexpr/diagnostic.hh new file mode 100644 index 00000000000..b962b8e12fb --- /dev/null +++ b/src/libexpr/diagnostic.hh @@ -0,0 +1,74 @@ +/// diagnostic.hh - This file declares records that related to nix diagnostic +#pragma once + +#include +#include + +#include "nixexpr.hh" + +namespace nix { + +/// The diagnostic +struct Diag +{ + PosIdx begin; + /// Which diagnostic + enum ID { +#define NIX_DIAG_ID(ID) ID, +#include "diagnostics-id.inc.hh" +#undef NIX_DIAG_ID + }; + + /// Tags + enum Tag { + DT_None, + DT_Unnecessary, + DT_Deprecated, + }; + + Diag(PosIdx begin) + : begin(begin) + { + } + + enum Severity { DL_Warning, DL_Error, DL_Note }; + + /// Additional information attached to some diagnostic. + /// elaborting the problem, + /// usaually points to a related piece of code. + std::vector notes; + + virtual Tag getTags() = 0; + virtual ID getID() = 0; + virtual Severity getServerity() = 0; + [[nodiscard]] virtual std::string format() const = 0; + + virtual ~Diag() = default; +}; + +struct DiagnosticEngine +{ + std::vector> Errors; + std::vector> Warnings; + + void add(std::unique_ptr D) + { + // Currently we just make the severity as-is + // we can use some flags to control (e.g. -Werror) + if (D->getServerity() == Diag::DL_Error) { + Errors.emplace_back(std::move(D)); + } else { + Warnings.emplace_back(std::move(D)); + } + } + + void checkRaise(const PosTable & positions) const + { + if (!Errors.empty()) { + const Diag * back = Errors.back().get(); + throw ParseError(ErrorInfo{.msg = back->format(), .errPos = positions[back->begin]}); + } + } +}; + +} // namespace nix diff --git a/src/libexpr/diagnostics-gen.nix b/src/libexpr/diagnostics-gen.nix new file mode 100644 index 00000000000..3f9cdbad8ed --- /dev/null +++ b/src/libexpr/diagnostics-gen.nix @@ -0,0 +1,54 @@ +# Diagnostics.nix, declaratively record nix diagnostics +let + mkDeclaration = + { name + , level + , body ? "" + , ctor ? "Diag${name}(PosIdx p): Diag(p) { }" + , format ? "return R\"(${message})\";" + , message + , tags ? "None" + }: '' + struct Diag${name} : Diag { + ${body} + ${ctor} + ID getID() override { + return DK_${name}; + } + Tag getTags() override { + return DT_${tags}; + } + Severity getServerity() override { + return DL_${level}; + } + std::string format() const override { + ${format} + } + }; + ''; + mkIDMacro = + { name + , ... + }: '' + NIX_DIAG_ID(DK_${name}) + ''; + + diagnostics = import ./diagnostics.nix; +in +{ + declarations = '' + /// Generated from diagnostic-gen.nix + #pragma once + #include "diagnostic.hh" + namespace nix { + ${builtins.concatStringsSep "" (map mkDeclaration diagnostics)} + } // namespace nix + ''; + idmacros = '' + /// Generated from diagnostic-gen.nix + #ifdef NIX_DIAG_ID + ${builtins.concatStringsSep "" (map mkIDMacro diagnostics)} + #endif + ''; +} + diff --git a/src/libexpr/diagnostics.nix b/src/libexpr/diagnostics.nix new file mode 100644 index 00000000000..e2a63325adb --- /dev/null +++ b/src/libexpr/diagnostics.nix @@ -0,0 +1,71 @@ +let + kinds = { + warning = "Warning"; + error = "Error"; + }; + tags = { + unnecessary = "Unnecessary"; + deprecated = "Deprecated"; + }; +in +[ + { + name = "PathHasTrailingSlash"; + level = kinds.error; + message = "path has a trailing slash"; + } + rec { + name = "InvalidInteger"; + level = kinds.error; + message = "invalid integer '%1%'"; + body = "std::string text;"; + ctor = ''Diag${name}(PosIdx p, std::string text): Diag(p), text(std::move(text)) { }''; + format = "return hintfmt(\"${message}\", text).str();"; + } + rec { + name = "InvalidFloat"; + level = kinds.error; + message = "invalid float '%1%'"; + body = "std::string text;"; + ctor = ''Diag${name}(PosIdx p, std::string text): Diag(p), text(std::move(text)) { }''; + format = "return hintfmt(\"${message}\", text).str();"; + } + { + name = "DynamicAttrsInLet"; + level = kinds.error; + message = "dynamic attributes not allowed in let"; + } + { + name = "URLLiteralsDisabled"; + level = kinds.error; + message = "URL literals are disabled"; + tags = tags.deprecated; + } + { + name = "URLLiterals"; + level = kinds.warning; + message = "using deprecated URL literal syntax"; + tags = tags.deprecated; + } + rec { + name = "HPathPure"; + level = kinds.error; + message = "home-path '%s' can not be resolved in pure mode"; + body = "std::string text;"; + ctor = ''Diag${name}(PosIdx p, std::string text): Diag(p), text(std::move(text)) { }''; + format = "return hintfmt(\"${message}\", text).str();"; + } + { + name = "InheritDynamic"; + level = kinds.error; + message = "dynamic attributes not allowed in inherit"; + } + rec { + name = "Syntax"; + level = kinds.error; + message = ""; + body = "std::string text;"; + ctor = ''Diag${name}(PosIdx p, std::string text): Diag(p), text(std::move(text)) { }''; + format = "return text;"; + } +] diff --git a/src/libexpr/lexer-prologue.cpp b/src/libexpr/lexer-prologue.cpp new file mode 100644 index 00000000000..48f9bb65c45 --- /dev/null +++ b/src/libexpr/lexer-prologue.cpp @@ -0,0 +1,93 @@ +#ifdef __clang__ +#pragma clang diagnostic ignored "-Wunneeded-internal-declaration" +#endif + +#include + +#include "nixexpr.hh" +#include "parser-tab.hh" + +using namespace nix; + +namespace nix { + +static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) +{ + return data->state.positions.add(data->origin, loc.first_line, loc.first_column); +} + +#define CUR_POS makeCurPos(*yylloc, data) + +// backup to recover from yyless(0) +thread_local YYLTYPE prev_yylloc; + +static void initLoc(YYLTYPE * loc) +{ + loc->first_line = loc->last_line = 1; + loc->first_column = loc->last_column = 1; +} + +static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) +{ + prev_yylloc = *loc; + + loc->first_line = loc->last_line; + loc->first_column = loc->last_column; + + for (size_t i = 0; i < len; i++) { + switch (*s++) { + case '\r': + if (*s == '\n') { /* cr/lf */ + i++; + s++; + } + /* fall through */ + case '\n': + ++loc->last_line; + loc->last_column = 1; + break; + default: + ++loc->last_column; + } + } +} + +// we make use of the fact that the parser receives a private copy of the input +// string and can munge around in it. +static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) +{ + char * result = s; + char * t = s; + char c; + // the input string is terminated with *two* NULs, so we can safely take + // *one* character after the one being checked against. + while ((c = *s++)) { + if (c == '\\') { + c = *s++; + if (c == 'n') + *t = '\n'; + else if (c == 'r') + *t = '\r'; + else if (c == 't') + *t = '\t'; + else + *t = c; + } else if (c == '\r') { + /* Normalise CR and CR/LF into LF. */ + *t = '\n'; + if (*s == '\n') + s++; /* cr/lf */ + } else + *t = c; + t++; + } + return {result, size_t(t - result)}; +} + +} + +#define YY_USER_INIT initLoc(yylloc) +#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng); + +#define PUSH_STATE(state) yy_push_state(state, yyscanner) +#define POP_STATE() yy_pop_state(yyscanner) diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index a3a8608d9a7..0a008770a4c 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -15,98 +15,7 @@ %{ -#ifdef __clang__ -#pragma clang diagnostic ignored "-Wunneeded-internal-declaration" -#endif - -#include - -#include "nixexpr.hh" -#include "parser-tab.hh" - -using namespace nix; - -namespace nix { - -static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) -{ - return data->state.positions.add(data->origin, loc.first_line, loc.first_column); -} - -#define CUR_POS makeCurPos(*yylloc, data) - -// backup to recover from yyless(0) -thread_local YYLTYPE prev_yylloc; - -static void initLoc(YYLTYPE * loc) -{ - loc->first_line = loc->last_line = 1; - loc->first_column = loc->last_column = 1; -} - -static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) -{ - prev_yylloc = *loc; - - loc->first_line = loc->last_line; - loc->first_column = loc->last_column; - - for (size_t i = 0; i < len; i++) { - switch (*s++) { - case '\r': - if (*s == '\n') { /* cr/lf */ - i++; - s++; - } - /* fall through */ - case '\n': - ++loc->last_line; - loc->last_column = 1; - break; - default: - ++loc->last_column; - } - } -} - - -// we make use of the fact that the parser receives a private copy of the input -// string and can munge around in it. -static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) -{ - char * result = s; - char * t = s; - char c; - // the input string is terminated with *two* NULs, so we can safely take - // *one* character after the one being checked against. - while ((c = *s++)) { - if (c == '\\') { - c = *s++; - if (c == 'n') *t = '\n'; - else if (c == 'r') *t = '\r'; - else if (c == 't') *t = '\t'; - else *t = c; - } - else if (c == '\r') { - /* Normalise CR and CR/LF into LF. */ - *t = '\n'; - if (*s == '\n') s++; /* cr/lf */ - } - else *t = c; - t++; - } - return {result, size_t(t - result)}; -} - - -} - -#define YY_USER_INIT initLoc(yylloc) -#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng); - -#define PUSH_STATE(state) yy_push_state(state, yyscanner) -#define POP_STATE() yy_pop_state(yyscanner) - +#include "lexer-prologue.cpp" %} @@ -153,20 +62,17 @@ or { return OR_KW; } try { yylval->n = boost::lexical_cast(yytext); } catch (const boost::bad_lexical_cast &) { - throw ParseError({ - .msg = hintfmt("invalid integer '%1%'", yytext), - .errPos = data->state.positions[CUR_POS], - }); + data->diags.add(std::make_unique(CUR_POS, yytext)); + yyterminate(); } return INT; } {FLOAT} { errno = 0; yylval->nf = strtod(yytext, 0); - if (errno != 0) - throw ParseError({ - .msg = hintfmt("invalid float '%1%'", yytext), - .errPos = data->state.positions[CUR_POS], - }); + if (errno != 0) { + data->diags.add(std::make_unique(CUR_POS, yytext)); + yyterminate(); + } return FLOAT; } @@ -292,10 +198,8 @@ or { return OR_KW; } {ANY} | <> { - throw ParseError({ - .msg = hintfmt("path has a trailing slash"), - .errPos = data->state.positions[CUR_POS], - }); + data->diags.add(std::make_unique(CUR_POS)); + yyterminate(); } {SPATH} { yylval->path = {yytext, (size_t) yyleng}; return SPATH; } diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index d243b9cec1d..bea8aafb2c0 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -4,6 +4,17 @@ libexpr_NAME = libnixexpr libexpr_DIR := $(d) +# Workaround for +# error: creating directory '/nix/var': Permission denied +# We are using nix itself to generate codes +# but it might not be able to run inside sandboxes (see comment above) +nixcmd = env NIX_LOCALSTATE_DIR=$(TMPDIR) \ + NIX_STORE_DIR=$(TMPDIR) \ + NIX_STATE_DIR=$(TMPDIR) \ + NIX_LOG_DIR=$(TMPDIR) \ + NIX_CONF_DIR=$(TMPDIR) nix + + libexpr_SOURCES := \ $(wildcard $(d)/*.cc) \ $(wildcard $(d)/value/*.cc) \ @@ -26,12 +37,23 @@ endif # because inline functions in libexpr's header files call libgc. libexpr_LDFLAGS_PROPAGATED = $(BDW_GC_LIBS) -libexpr_ORDER_AFTER := $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh +libexpr_ORDER_AFTER := $(d)/parser-tab.cc \ + $(d)/parser-tab.hh \ + $(d)/lexer-tab.cc \ + $(d)/lexer-tab.hh \ + $(d)/diagnostics.inc.hh \ + $(d)/diagnostics-id.inc.hh + +$(d)/diagnostics-id.inc.hh: $(d)/diagnostics-gen.nix $(d)/diagnostics.nix + $(trace-gen) $(nixcmd) --experimental-features "nix-command" eval --raw --file $< idmacros > $@ + +$(d)/diagnostics.inc.hh: $(d)/diagnostics-gen.nix $(d)/diagnostics.nix + $(trace-gen) $(nixcmd) --experimental-features "nix-command" eval --raw --file $< declarations > $@ -$(d)/parser-tab.cc $(d)/parser-tab.hh: $(d)/parser.y +$(d)/parser-tab.cc $(d)/parser-tab.hh: $(d)/parser.y $(d)/parser-prologue.cpp $(d)/parser-epilogue.cpp $(trace-gen) bison -v -o $(libexpr_DIR)/parser-tab.cc $< -d -$(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l +$(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l $(d)/lexer-prologue.cpp $(trace-gen) flex --outfile $(libexpr_DIR)/lexer-tab.cc --header-file=$(libexpr_DIR)/lexer-tab.hh $< clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh diff --git a/src/libexpr/parser-epilogue.cpp b/src/libexpr/parser-epilogue.cpp new file mode 100644 index 00000000000..9b69e5d6543 --- /dev/null +++ b/src/libexpr/parser-epilogue.cpp @@ -0,0 +1,177 @@ +#include +#include +#include +#include + +#include "parser-tab.hh" + +#include "lexer-tab.hh" + +#include "eval.hh" +#include "filetransfer.hh" +#include "fetchers.hh" +#include "store-api.hh" +#include "flake/flake.hh" + +namespace nix { + +unsigned long Expr::nrExprs = 0; + +Expr * EvalState::parse( + char * text, size_t length, Pos::Origin origin, const SourcePath & basePath, std::shared_ptr & staticEnv) +{ + yyscan_t scanner; + ParseData data{ + .state = *this, + .symbols = symbols, + .basePath = basePath, + .origin = {origin}, + }; + + yylex_init(&scanner); + yy_scan_buffer(text, length, scanner); + yyparse(scanner, &data); + yylex_destroy(scanner); + + data.diags.checkRaise(data.state.positions); + + data.result->bindVars(*this, staticEnv); + + return data.result; +} + +SourcePath resolveExprPath(const SourcePath & path) +{ + /* If `path' is a symlink, follow it. This is so that relative + path references work. */ + auto path2 = path.resolveSymlinks(); + + /* If `path' refers to a directory, append `/default.nix'. */ + if (path2.lstat().type == InputAccessor::tDirectory) + return path2 + "default.nix"; + + return path2; +} + +Expr * EvalState::parseExprFromFile(const SourcePath & path) +{ + return parseExprFromFile(path, staticBaseEnv); +} + +Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv) +{ + auto buffer = path.readFile(); + // readFile hopefully have left some extra space for terminators + buffer.append("\0\0", 2); + return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv); +} + +Expr * +EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr & staticEnv) +{ + auto s = make_ref(std::move(s_)); + s->append("\0\0", 2); + return parse(s->data(), s->size(), Pos::String{.source = s}, basePath, staticEnv); +} + +Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath) +{ + return parseExprFromString(std::move(s), basePath, staticBaseEnv); +} + +Expr * EvalState::parseStdin() +{ + // Activity act(*logger, lvlTalkative, "parsing standard input"); + auto buffer = drainFD(0); + // drainFD should have left some extra space for terminators + buffer.append("\0\0", 2); + auto s = make_ref(std::move(buffer)); + return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv); +} + +SourcePath EvalState::findFile(const std::string_view path) +{ + return findFile(searchPath, path); +} + +SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos) +{ + for (auto & i : searchPath.elements) { + auto suffixOpt = i.prefix.suffixIfPotentialMatch(path); + + if (!suffixOpt) + continue; + auto suffix = *suffixOpt; + + auto rOpt = resolveSearchPathPath(i.path); + if (!rOpt) + continue; + auto r = *rOpt; + + Path res = suffix == "" ? r : concatStrings(r, "/", suffix); + if (pathExists(res)) + return CanonPath(canonPath(res)); + } + + if (hasPrefix(path, "nix/")) + return CanonPath(concatStrings(corepkgsPrefix, path.substr(4))); + + debugThrow( + ThrownError( + {.msg = hintfmt( + evalSettings.pureEval + ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" + : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", + path), + .errPos = positions[pos]}), + 0, 0); +} + +std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0) +{ + auto & value = value0.s; + auto i = searchPathResolved.find(value); + if (i != searchPathResolved.end()) + return i->second; + + std::optional res; + + if (EvalSettings::isPseudoUrl(value)) { + try { + auto storePath = + fetchers::downloadTarball(store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath; + res = {store->toRealPath(storePath)}; + } catch (FileTransferError & e) { + logWarning({.msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)}); + res = std::nullopt; + } + } + + else if (hasPrefix(value, "flake:")) { + experimentalFeatureSettings.require(Xp::Flakes); + auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); + debug("fetching flake search path element '%s''", value); + auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; + res = {store->toRealPath(storePath)}; + } + + else { + auto path = absPath(value); + if (pathExists(path)) + res = {path}; + else { + logWarning({.msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value)}); + res = std::nullopt; + } + } + + if (res) + debug("resolved search path element '%s' to '%s'", value, *res); + else + debug("failed to resolve search path element '%s'", value); + + searchPathResolved[value] = res; + return res; +} + +} diff --git a/src/libexpr/parser-prologue.cpp b/src/libexpr/parser-prologue.cpp new file mode 100644 index 00000000000..759c3b6dab0 --- /dev/null +++ b/src/libexpr/parser-prologue.cpp @@ -0,0 +1,232 @@ + +#include "diagnostics.inc.hh" +#include "parser-tab.hh" +#include "lexer-tab.hh" + +YY_DECL; + +using namespace nix; + +namespace nix { + +static void dupAttr(const EvalState & state, const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos) +{ + throw ParseError( + {.msg = hintfmt( + "attribute '%1%' already defined at %2%", showAttrPath(state.symbols, attrPath), state.positions[prevPos]), + .errPos = state.positions[pos]}); +} + +static void dupAttr(const EvalState & state, Symbol attr, const PosIdx pos, const PosIdx prevPos) +{ + throw ParseError( + {.msg = hintfmt("attribute '%1%' already defined at %2%", state.symbols[attr], state.positions[prevPos]), + .errPos = state.positions[pos]}); +} + +static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos, const nix::EvalState & state) +{ + AttrPath::iterator i; + // All attrpaths have at least one attr + assert(!attrPath.empty()); + // Checking attrPath validity. + // =========================== + for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { + if (i->symbol) { + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); + if (j != attrs->attrs.end()) { + if (!j->second.inherited) { + ExprAttrs * attrs2 = dynamic_cast(j->second.e); + if (!attrs2) + dupAttr(state, attrPath, pos, j->second.pos); + attrs = attrs2; + } else + dupAttr(state, attrPath, pos, j->second.pos); + } else { + ExprAttrs * nested = new ExprAttrs; + attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos); + attrs = nested; + } + } else { + ExprAttrs * nested = new ExprAttrs; + attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); + attrs = nested; + } + } + // Expr insertion. + // ========================== + if (i->symbol) { + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); + if (j != attrs->attrs.end()) { + // This attr path is already defined. However, if both + // e and the expr pointed by the attr path are two attribute sets, + // we want to merge them. + // Otherwise, throw an error. + auto ae = dynamic_cast(e); + auto jAttrs = dynamic_cast(j->second.e); + if (jAttrs && ae) { + for (auto & ad : ae->attrs) { + auto j2 = jAttrs->attrs.find(ad.first); + if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. + dupAttr(state, ad.first, j2->second.pos, ad.second.pos); + jAttrs->attrs.emplace(ad.first, ad.second); + } + jAttrs->dynamicAttrs.insert( + jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); + } else { + dupAttr(state, attrPath, pos, j->second.pos); + } + } else { + // This attr path is not defined. Let's create it. + attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos)); + e->setName(i->symbol); + } + } else { + attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); + } +} + +static Formals * toFormals(ParseData & data, ParserFormals * formals, PosIdx pos = noPos, Symbol arg = {}) +{ + std::sort(formals->formals.begin(), formals->formals.end(), [](const auto & a, const auto & b) { + return std::tie(a.name, a.pos) < std::tie(b.name, b.pos); + }); + + std::optional> duplicate; + for (size_t i = 0; i + 1 < formals->formals.size(); i++) { + if (formals->formals[i].name != formals->formals[i + 1].name) + continue; + std::pair thisDup{formals->formals[i].name, formals->formals[i + 1].pos}; + duplicate = std::min(thisDup, duplicate.value_or(thisDup)); + } + if (duplicate) + throw ParseError( + {.msg = hintfmt("duplicate formal function argument '%1%'", data.symbols[duplicate->first]), + .errPos = data.state.positions[duplicate->second]}); + + Formals result; + result.ellipsis = formals->ellipsis; + result.formals = std::move(formals->formals); + + if (arg && result.has(arg)) + throw ParseError( + {.msg = hintfmt("duplicate formal function argument '%1%'", data.symbols[arg]), + .errPos = data.state.positions[pos]}); + + delete formals; + return new Formals(std::move(result)); +} + +static Expr * stripIndentation( + const PosIdx pos, SymbolTable & symbols, std::vector>> && es) +{ + if (es.empty()) + return new ExprString(""); + + /* Figure out the minimum indentation. Note that by design + whitespace-only final lines are not taken into account. (So + the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ + bool atStartOfLine = true; /* = seen only whitespace in the current line */ + size_t minIndent = 1000000; + size_t curIndent = 0; + for (auto & [i_pos, i] : es) { + auto * str = std::get_if(&i); + if (!str || !str->hasIndentation) { + /* Anti-quotations and escaped characters end the current start-of-line whitespace. */ + if (atStartOfLine) { + atStartOfLine = false; + if (curIndent < minIndent) + minIndent = curIndent; + } + continue; + } + for (size_t j = 0; j < str->l; ++j) { + if (atStartOfLine) { + if (str->p[j] == ' ') + curIndent++; + else if (str->p[j] == '\n') { + /* Empty line, doesn't influence minimum + indentation. */ + curIndent = 0; + } else { + atStartOfLine = false; + if (curIndent < minIndent) + minIndent = curIndent; + } + } else if (str->p[j] == '\n') { + atStartOfLine = true; + curIndent = 0; + } + } + } + + /* Strip spaces from each line. */ + auto * es2 = new std::vector>; + atStartOfLine = true; + size_t curDropped = 0; + size_t n = es.size(); + auto i = es.begin(); + const auto trimExpr = [&](Expr * e) { + atStartOfLine = false; + curDropped = 0; + es2->emplace_back(i->first, e); + }; + const auto trimString = [&](const StringToken & t) { + std::string s2; + for (size_t j = 0; j < t.l; ++j) { + if (atStartOfLine) { + if (t.p[j] == ' ') { + if (curDropped++ >= minIndent) + s2 += t.p[j]; + } else if (t.p[j] == '\n') { + curDropped = 0; + s2 += t.p[j]; + } else { + atStartOfLine = false; + curDropped = 0; + s2 += t.p[j]; + } + } else { + s2 += t.p[j]; + if (t.p[j] == '\n') + atStartOfLine = true; + } + } + + /* Remove the last line if it is empty and consists only of + spaces. */ + if (n == 1) { + std::string::size_type p = s2.find_last_of('\n'); + if (p != std::string::npos && s2.find_first_not_of(' ', p + 1) == std::string::npos) + s2 = std::string(s2, 0, p + 1); + } + + es2->emplace_back(i->first, new ExprString(std::move(s2))); + }; + for (; i != es.end(); ++i, --n) { + std::visit(overloaded{trimExpr, trimString}, i->second); + } + + /* If this is a single string, then don't do a concatenation. */ + if (es2->size() == 1 && dynamic_cast((*es2)[0].second)) { + auto * const result = (*es2)[0].second; + delete es2; + return result; + } + return new ExprConcatStrings(pos, true, es2); +} + +static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) +{ + return data->state.positions.add(data->origin, loc.first_line, loc.first_column); +} + +#define CUR_POS makeCurPos(*yylocp, data) + +} + +void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) +{ + PosIdx pos = makeCurPos(*loc, data); + data->diags.add(std::make_unique(pos, error)); +} diff --git a/src/libexpr/parser-requires.hh b/src/libexpr/parser-requires.hh new file mode 100644 index 00000000000..d185a7c51a6 --- /dev/null +++ b/src/libexpr/parser-requires.hh @@ -0,0 +1,53 @@ + +#ifndef BISON_HEADER +#define BISON_HEADER + +#include + +#include "diagnostic.hh" +#include "diagnostics.inc.hh" +#include "util.hh" + +#include "nixexpr.hh" +#include "eval.hh" +#include "eval-settings.hh" +#include "globals.hh" + +namespace nix { + +struct ParseData +{ + EvalState & state; + SymbolTable & symbols; + Expr * result; + SourcePath basePath; + PosTable::Origin origin; + + /// Stores diagnostics while parsing this input + DiagnosticEngine diags; +}; + +struct ParserFormals +{ + std::vector formals; + bool ellipsis = false; +}; + +} + +// using C a struct allows us to avoid having to define the special +// members that using string_view here would implicitly delete. +struct StringToken +{ + const char * p; + size_t l; + bool hasIndentation; + operator std::string_view() const + { + return {p, l}; + } +}; + +#define YY_DECL int yylex(YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParseData * data) + +#endif // BISON_HEADER diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 70228e1e273..bbc097e6fdd 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -12,300 +12,10 @@ %expect-rr 1 %code requires { - -#ifndef BISON_HEADER -#define BISON_HEADER - -#include - -#include "util.hh" - -#include "nixexpr.hh" -#include "eval.hh" -#include "eval-settings.hh" -#include "globals.hh" - -namespace nix { - - struct ParseData - { - EvalState & state; - SymbolTable & symbols; - Expr * result; - SourcePath basePath; - PosTable::Origin origin; - std::optional error; - }; - - struct ParserFormals { - std::vector formals; - bool ellipsis = false; - }; - -} - -// using C a struct allows us to avoid having to define the special -// members that using string_view here would implicitly delete. -struct StringToken { - const char * p; - size_t l; - bool hasIndentation; - operator std::string_view() const { return {p, l}; } -}; - -#define YY_DECL int yylex \ - (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParseData * data) - -#endif - +#include "parser-requires.hh" } - %{ - -#include "parser-tab.hh" -#include "lexer-tab.hh" - -YY_DECL; - -using namespace nix; - - -namespace nix { - - -static void dupAttr(const EvalState & state, const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos) -{ - throw ParseError({ - .msg = hintfmt("attribute '%1%' already defined at %2%", - showAttrPath(state.symbols, attrPath), state.positions[prevPos]), - .errPos = state.positions[pos] - }); -} - -static void dupAttr(const EvalState & state, Symbol attr, const PosIdx pos, const PosIdx prevPos) -{ - throw ParseError({ - .msg = hintfmt("attribute '%1%' already defined at %2%", state.symbols[attr], state.positions[prevPos]), - .errPos = state.positions[pos] - }); -} - - -static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, - Expr * e, const PosIdx pos, const nix::EvalState & state) -{ - AttrPath::iterator i; - // All attrpaths have at least one attr - assert(!attrPath.empty()); - // Checking attrPath validity. - // =========================== - for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { - if (i->symbol) { - ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); - if (j != attrs->attrs.end()) { - if (!j->second.inherited) { - ExprAttrs * attrs2 = dynamic_cast(j->second.e); - if (!attrs2) dupAttr(state, attrPath, pos, j->second.pos); - attrs = attrs2; - } else - dupAttr(state, attrPath, pos, j->second.pos); - } else { - ExprAttrs * nested = new ExprAttrs; - attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos); - attrs = nested; - } - } else { - ExprAttrs *nested = new ExprAttrs; - attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); - attrs = nested; - } - } - // Expr insertion. - // ========================== - if (i->symbol) { - ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); - if (j != attrs->attrs.end()) { - // This attr path is already defined. However, if both - // e and the expr pointed by the attr path are two attribute sets, - // we want to merge them. - // Otherwise, throw an error. - auto ae = dynamic_cast(e); - auto jAttrs = dynamic_cast(j->second.e); - if (jAttrs && ae) { - for (auto & ad : ae->attrs) { - auto j2 = jAttrs->attrs.find(ad.first); - if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. - dupAttr(state, ad.first, j2->second.pos, ad.second.pos); - jAttrs->attrs.emplace(ad.first, ad.second); - } - jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); - } else { - dupAttr(state, attrPath, pos, j->second.pos); - } - } else { - // This attr path is not defined. Let's create it. - attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos)); - e->setName(i->symbol); - } - } else { - attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); - } -} - - -static Formals * toFormals(ParseData & data, ParserFormals * formals, - PosIdx pos = noPos, Symbol arg = {}) -{ - std::sort(formals->formals.begin(), formals->formals.end(), - [] (const auto & a, const auto & b) { - return std::tie(a.name, a.pos) < std::tie(b.name, b.pos); - }); - - std::optional> duplicate; - for (size_t i = 0; i + 1 < formals->formals.size(); i++) { - if (formals->formals[i].name != formals->formals[i + 1].name) - continue; - std::pair thisDup{formals->formals[i].name, formals->formals[i + 1].pos}; - duplicate = std::min(thisDup, duplicate.value_or(thisDup)); - } - if (duplicate) - throw ParseError({ - .msg = hintfmt("duplicate formal function argument '%1%'", data.symbols[duplicate->first]), - .errPos = data.state.positions[duplicate->second] - }); - - Formals result; - result.ellipsis = formals->ellipsis; - result.formals = std::move(formals->formals); - - if (arg && result.has(arg)) - throw ParseError({ - .msg = hintfmt("duplicate formal function argument '%1%'", data.symbols[arg]), - .errPos = data.state.positions[pos] - }); - - delete formals; - return new Formals(std::move(result)); -} - - -static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols, - std::vector>> && es) -{ - if (es.empty()) return new ExprString(""); - - /* Figure out the minimum indentation. Note that by design - whitespace-only final lines are not taken into account. (So - the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ - bool atStartOfLine = true; /* = seen only whitespace in the current line */ - size_t minIndent = 1000000; - size_t curIndent = 0; - for (auto & [i_pos, i] : es) { - auto * str = std::get_if(&i); - if (!str || !str->hasIndentation) { - /* Anti-quotations and escaped characters end the current start-of-line whitespace. */ - if (atStartOfLine) { - atStartOfLine = false; - if (curIndent < minIndent) minIndent = curIndent; - } - continue; - } - for (size_t j = 0; j < str->l; ++j) { - if (atStartOfLine) { - if (str->p[j] == ' ') - curIndent++; - else if (str->p[j] == '\n') { - /* Empty line, doesn't influence minimum - indentation. */ - curIndent = 0; - } else { - atStartOfLine = false; - if (curIndent < minIndent) minIndent = curIndent; - } - } else if (str->p[j] == '\n') { - atStartOfLine = true; - curIndent = 0; - } - } - } - - /* Strip spaces from each line. */ - auto * es2 = new std::vector>; - atStartOfLine = true; - size_t curDropped = 0; - size_t n = es.size(); - auto i = es.begin(); - const auto trimExpr = [&] (Expr * e) { - atStartOfLine = false; - curDropped = 0; - es2->emplace_back(i->first, e); - }; - const auto trimString = [&] (const StringToken & t) { - std::string s2; - for (size_t j = 0; j < t.l; ++j) { - if (atStartOfLine) { - if (t.p[j] == ' ') { - if (curDropped++ >= minIndent) - s2 += t.p[j]; - } - else if (t.p[j] == '\n') { - curDropped = 0; - s2 += t.p[j]; - } else { - atStartOfLine = false; - curDropped = 0; - s2 += t.p[j]; - } - } else { - s2 += t.p[j]; - if (t.p[j] == '\n') atStartOfLine = true; - } - } - - /* Remove the last line if it is empty and consists only of - spaces. */ - if (n == 1) { - std::string::size_type p = s2.find_last_of('\n'); - if (p != std::string::npos && s2.find_first_not_of(' ', p + 1) == std::string::npos) - s2 = std::string(s2, 0, p + 1); - } - - es2->emplace_back(i->first, new ExprString(std::move(s2))); - }; - for (; i != es.end(); ++i, --n) { - std::visit(overloaded { trimExpr, trimString }, i->second); - } - - /* If this is a single string, then don't do a concatenation. */ - if (es2->size() == 1 && dynamic_cast((*es2)[0].second)) { - auto *const result = (*es2)[0].second; - delete es2; - return result; - } - return new ExprConcatStrings(pos, true, es2); -} - - -static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) -{ - return data->state.positions.add(data->origin, loc.first_line, loc.first_column); -} - -#define CUR_POS makeCurPos(*yylocp, data) - - -} - - -void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) -{ - data->error = { - .msg = hintfmt(error), - .errPos = data->state.positions[makeCurPos(*loc, data)] - }; -} - - +#include "parser-prologue.cpp" %} %union { @@ -387,11 +97,10 @@ expr_function | WITH expr ';' expr_function { $$ = new ExprWith(CUR_POS, $2, $4); } | LET binds IN expr_function - { if (!$2->dynamicAttrs.empty()) - throw ParseError({ - .msg = hintfmt("dynamic attributes not allowed in let"), - .errPos = data->state.positions[CUR_POS] - }); + { + if (!$2->dynamicAttrs.empty()) { + data->diags.add(std::make_unique(CUR_POS)); + } $$ = new ExprLet($2, $4); } | expr_if @@ -477,11 +186,11 @@ expr_simple } | URI { static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals); - if (noURLLiterals) - throw ParseError({ - .msg = hintfmt("URL literals are disabled"), - .errPos = data->state.positions[CUR_POS] - }); + if (noURLLiterals) { + data->diags.add(std::make_unique(CUR_POS)); + } else { + data->diags.add(std::make_unique(CUR_POS)); + } $$ = new ExprString(std::string($1)); } | '(' expr ')' { $$ = $2; } @@ -524,10 +233,7 @@ path_start } | HPATH { if (evalSettings.pureEval) { - throw Error( - "the path '%s' can not be resolved in pure mode", - std::string_view($1.p, $1.l) - ); + data->diags.add(std::make_unique(CUR_POS, std::string($1.p + 1, $1.l - 1))); } Path path(getHome() + std::string($1.p + 1, $1.l - 1)); $$ = new ExprPath(std::move(path)); @@ -574,10 +280,7 @@ attrs $$->push_back(AttrName(data->symbols.create(str->s))); delete str; } else - throw ParseError({ - .msg = hintfmt("dynamic attributes not allowed in inherit"), - .errPos = data->state.positions[makeCurPos(@2, data)] - }); + data->diags.add(std::make_unique(CUR_POS)); } | { $$ = new AttrPath; } ; @@ -638,189 +341,4 @@ formal %% - -#include -#include -#include -#include - -#include "eval.hh" -#include "filetransfer.hh" -#include "fetchers.hh" -#include "store-api.hh" -#include "flake/flake.hh" - - -namespace nix { - -unsigned long Expr::nrExprs = 0; - -Expr * EvalState::parse( - char * text, - size_t length, - Pos::Origin origin, - const SourcePath & basePath, - std::shared_ptr & staticEnv) -{ - yyscan_t scanner; - ParseData data { - .state = *this, - .symbols = symbols, - .basePath = basePath, - .origin = {origin}, - }; - - yylex_init(&scanner); - yy_scan_buffer(text, length, scanner); - int res = yyparse(scanner, &data); - yylex_destroy(scanner); - - if (res) throw ParseError(data.error.value()); - - data.result->bindVars(*this, staticEnv); - - return data.result; -} - - -SourcePath resolveExprPath(const SourcePath & path) -{ - /* If `path' is a symlink, follow it. This is so that relative - path references work. */ - auto path2 = path.resolveSymlinks(); - - /* If `path' refers to a directory, append `/default.nix'. */ - if (path2.lstat().type == InputAccessor::tDirectory) - return path2 + "default.nix"; - - return path2; -} - - -Expr * EvalState::parseExprFromFile(const SourcePath & path) -{ - return parseExprFromFile(path, staticBaseEnv); -} - - -Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv) -{ - auto buffer = path.readFile(); - // readFile hopefully have left some extra space for terminators - buffer.append("\0\0", 2); - return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv); -} - - -Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr & staticEnv) -{ - auto s = make_ref(std::move(s_)); - s->append("\0\0", 2); - return parse(s->data(), s->size(), Pos::String{.source = s}, basePath, staticEnv); -} - - -Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath) -{ - return parseExprFromString(std::move(s), basePath, staticBaseEnv); -} - - -Expr * EvalState::parseStdin() -{ - //Activity act(*logger, lvlTalkative, "parsing standard input"); - auto buffer = drainFD(0); - // drainFD should have left some extra space for terminators - buffer.append("\0\0", 2); - auto s = make_ref(std::move(buffer)); - return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv); -} - - -SourcePath EvalState::findFile(const std::string_view path) -{ - return findFile(searchPath, path); -} - - -SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos) -{ - for (auto & i : searchPath.elements) { - auto suffixOpt = i.prefix.suffixIfPotentialMatch(path); - - if (!suffixOpt) continue; - auto suffix = *suffixOpt; - - auto rOpt = resolveSearchPathPath(i.path); - if (!rOpt) continue; - auto r = *rOpt; - - Path res = suffix == "" ? r : concatStrings(r, "/", suffix); - if (pathExists(res)) return CanonPath(canonPath(res)); - } - - if (hasPrefix(path, "nix/")) - return CanonPath(concatStrings(corepkgsPrefix, path.substr(4))); - - debugThrow(ThrownError({ - .msg = hintfmt(evalSettings.pureEval - ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" - : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", - path), - .errPos = positions[pos] - }), 0, 0); -} - - -std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0) -{ - auto & value = value0.s; - auto i = searchPathResolved.find(value); - if (i != searchPathResolved.end()) return i->second; - - std::optional res; - - if (EvalSettings::isPseudoUrl(value)) { - try { - auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath; - res = { store->toRealPath(storePath) }; - } catch (FileTransferError & e) { - logWarning({ - .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) - }); - res = std::nullopt; - } - } - - else if (hasPrefix(value, "flake:")) { - experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); - debug("fetching flake search path element '%s''", value); - auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; - res = { store->toRealPath(storePath) }; - } - - else { - auto path = absPath(value); - if (pathExists(path)) - res = { path }; - else { - logWarning({ - .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value) - }); - res = std::nullopt; - } - } - - if (res) - debug("resolved search path element '%s' to '%s'", value, *res); - else - debug("failed to resolve search path element '%s'", value); - - searchPathResolved[value] = res; - return res; -} - - -} +#include "parser-epilogue.cpp"