ubgpsuite/lonetix/include/df/lexer.h

// SPDX-License-Identifier: LGPL-3.0-or-later

/**
 * \file lexer.h
 *
 * C-compliant non-allocating UTF-8 text lexer.
 *
 * \author Lorenzo Cogotti
 * \copyright The DoubleFourteen Code Forge (C) All Rights Reserved
 */

#ifndef DF_LEXER_H_
#define DF_LEXER_H_

#include "utf/utfdef.h"

/// Maximum allowed token length inside text parsed by `Lex`.
#define MAXTOKLEN 256

/// String token type
#define TT_STRING  U16_C(1)
/// Literal token type
#define TT_LITERAL U16_C(2)
/// Numeric token type
#define TT_NUMBER  U16_C(3)
/// Token type for names or identifiers
#define TT_NAME    U16_C(4)
/// Punctuation token type
#define TT_PUNCT   U16_C(5)

/**
 * Token subtype flags for `TT_NUMBER`
 *
 * @{
 */

#define TT_INT         BIT(0)   ///< integer
#define TT_DEC         BIT(1)   ///< decimal number
#define TT_HEX         BIT(2)   ///< hexadecimal number
#define TT_OCT         BIT(3)   ///< octal number
#define TT_BIN         BIT(4)   ///< binary number
#define TT_LONG        BIT(5)   ///< long int
#define TT_LLONG       BIT(6)   ///< long long int
#define TT_UNSIGNED    BIT(7)   ///< unsigned int
#define TT_FLOAT       BIT(8)   ///< floating point number
#define TT_SINGLE_PREC BIT(9)   ///< float
#define TT_DOUBLE_PREC BIT(10)  ///< double
#define TT_EXT_PREC    BIT(11)  ///< long double
#define TT_INF         BIT(12)  ///< infinite 1.#INF
#define TT_INDEF       BIT(13)  ///< indefinite 1.#IND
#define TT_NAN         BIT(14)  ///< NaN
#define TT_IPADDR      BIT(15)  ///< ip address (address may still be ill-formed, e.g. `102948.22.999.1`)
#define TT_IPV4        BIT(16)  ///< ipv4 address format
#define TT_IPV6        BIT(17)  ///< ipv6 address format
#define TT_IPV6LIT     BIT(18)  ///< ipv6 address is expressed as literal (e.g. `[2001:db8:a::123]`)
#define TT_IPV6ZONE    BIT(19)  ///< ipv6 address contains a zone index/string (e.g. `fe80::1ff:fe23:4567:890a%3`)
#define TT_IPPORT      BIT(20)  ///< ip address includes a port

/** @} */

/**
 * Token flags
 *
 * @{
 */

/// Indicates `Tok` originally exceeded `MAXTOKLEN` and was consequently truncated.
#define TT_TRUNC BIT(15)

/** @} */

/// Lexer punctuation token descriptor (text -> token `subtype`).
typedef struct Punctuation Punctuation;
struct Punctuation {
	const char *p;   ///< NULL for last element in punctuation list.
	Uint32      id;  ///< Puntuation identifier (returned in `Tok->subtype`)
};

// punctuation ids
#define P_RSHIFT_ASSIGN 1
#define P_LSHIFT_ASSIGN 2
#define P_PARMS         3
#define P_PRECOMPMERGE  4

#define P_LOGIC_AND  5
#define P_LOGIC_OR   6
#define P_LOGIC_GEQ  7
#define P_LOGIC_LEQ  8
#define P_LOGIC_EQ   9
#define P_LOGIC_UNEQ 10

#define P_MUL_ASSIGN 11
#define P_DIV_ASSIGN 12
#define P_MOD_ASSIGN 13
#define P_ADD_ASSIGN 14
#define P_SUB_ASSIGN 15
#define P_INC        16
#define P_DEC        17

#define P_BIN_AND_ASSIGN 18
#define P_BIN_OR_ASSIGN  19
#define P_BIN_XOR_ASSIGN 20
#define P_RSHIFT         21
#define P_LSHIFT         22

#define P_POINTERREF 23
#define P_MUL        24
#define P_DIV        25
#define P_MOD        26
#define P_ADD        27
#define P_SUB        28
#define P_ASSIGN     29

#define P_BIN_AND 30
#define P_BIN_OR  31
#define P_BIN_XOR 32
#define P_BIN_NOT 33

#define P_LOGIC_NOT     34
#define P_LOGIC_GREATER 35
#define P_LOGIC_LESS    36

#define P_REF          37
#define P_COMMA        38
#define P_SEMICOLON    39
#define P_COLON        40
#define P_QUESTIONMARK 41

#define P_PARENOPEN      42
#define P_PARENCLOSE     43
#define P_BRACEOPEN      44
#define P_BRACECLOSE     45
#define P_SQBRACKETOPEN  46
#define P_SQBRACKETCLOSE 47
#define P_BACKSLASH      48

#define P_PRECOMP 49
#define P_DOLLAR  50

/**
 * \brief Token returned by `Lex`.
 *
 * Contains token text and information.
 */
typedef struct Tok Tok;
struct Tok {
	Uint16 type;
	Uint16 flags;
	Uint32 subtype;

	unsigned linesCrossed;
	unsigned spacesBeforeToken;
	unsigned line;

	long long intvalue;
	double    floatvalue;

	Tok *nextToken;

	char text[MAXTOKLEN];  // NOTE: last element to allow partial allocation
};

/// Disregard lexer errors
#define L_NOERR  BIT(0)
/// Disregard lexer warnings
#define L_NOWARN BIT(1)
/// Disregard both errors and warnings
#define L_QUIET (L_NOERR | L_NOWARN)
/// Use console colors when reporting errors and warnings
#define L_COLORED BIT(2)
/// Parse all tokens as strings, instead of breaking them using full-fledged C rules
#define L_STRONLY BIT(3)
/// Allow file paths within tokens
#define L_ALLOWPATHS BIT(4)
/// Do not allow escapes within strings
#define L_NOSTRESC BIT(5)
/// Do not concatenate consecutive strings
#define L_NOSTRCAT BIT(6)
/// Concatenate strings separated by a backslash+newline
#define L_ALLOWBACKSLASHSTRCAT BIT(7)
/// Allow multichar literals
#define L_ALLOWMULTICHARLIT    BIT(8)
/// Accepts IP addresses (parsed as `TT_NUMBER`)
#define L_ALLOWIPADDR          BIT(9)
/// IP addresses with port numbers, IPv6 literals or zone ids won't be accepted,
/// only meaningful if used with `L_ALLOWIPADDR`.
#define L_PLAINIPADDRONLY      BIT(10)
/// Allow special floating point exception tokens (0.#INF, 0.#IND).
#define L_ALLOWFLOATEXC        BIT(10)
/// Allow truncating tokens exceeding `MAXTOKLEN`.
#define L_ALLOWTRUNC           BIT(11)
/// Do not search base `#include` paths (used by PC library).
#define L_NOBASEINCLUDES       BIT(12)

/// Special callback, invokes immediate program termination after reporting a lexer message
#define LEX_QUIT  ((void (*)(Lex *, const char *, void *)) -1)
/// Special callback, makes the lexer ignore the the warning or error
/// (same behavior as `L_NOERR` and `L_NOWARN`, but as an explicit callback).
#define LEX_IGN   ((void (*)(Lex *, const char *, void *))  0)
/// Special callback, makes the lexer print an error or warning message to `stderr`,
/// doesn't terminate execution.
#define LEX_WARN  ((void (*)(Lex *, const char *, void *))  1)

/**
 * \brief A lexer, breaks text into single tokens, keeping track of the current position.
 *
 * \note This struct should be considered opaque.
 */
typedef struct Lex Lex;
struct Lex {
	char    *pos, *lim;
	unsigned line;
	Uint16   flags;
	Boolean8 hasError;
	Boolean8 hasBufferedToken;
	Rune     nr;

	const Punctuation *puncts;

	void  *obj;
	void (*Error)(Lex *, const char *, void *);
	void (*Warn)(Lex *, const char *, void *);

	Lex *nextLexer;

	Tok  buf;
	char name[MAXTOKLEN];
};

/// Register callbacks for lexer warning and error triggers.
FORCE_INLINE void SetLexerErrorFunc(Lex   *p,
                                    void (*errf)(Lex *, const char *, void *),
                                    void (*warnf)(Lex *, const char *, void *),
                                    void  *obj)
{
	p->Error = errf;
	p->Warn  = warnf;
	p->obj   = obj;
}

/**
 * \brief Set parsing session name and initial line number.
 *
 * \param [out] p    A lexer, must not be `NULL`
 * \param [in]  name Name for this parsing session
 * \param [in]  line Initial line number, 0 is implicitly changed to 1
 */
void BeginLexerSession(Lex *p, const char *name, unsigned line);

/**
 * \brief Setup lexer to parse text, sized.
 *
 * \param [out] p    A lexer, must not be `NULL`
 * \param [in]  text Text to be parsed, must have at least `n` chars
 * \param [in]  n    Number of chars in `text`
 */
void SetLexerTextn(Lex *p, const char *text, size_t n);

/**
 * \brief Setup lexer to parse text.
 *
 * \param [out] p    A lexer, must not be `NULL`
 * \param [in]  text `NUL` terminated text to be parsed
 */
FORCE_INLINE void SetLexerText(Lex *p, const char *text)
{
	EXTERNC size_t strlen(const char *);

	SetLexerTextn(p, text, strlen(text));
}

/**
 * \brief Change lexer flags.
 *
 * \param [out] p     A lexer, must not be `NULL`
 * \param [in]  flags New flags for the lexer
 */
FORCE_INLINE void SetLexerFlags(Lex *p, unsigned flags)
{
	p->flags = flags;
}

/// Retrieve current lexer flags.
FORCE_INLINE unsigned GetLexerFlags(Lex *p)
{
	return p->flags;
}

/// Trigger an error over a lexer.
CHECK_PRINTF(2, 3) void LexerError(Lex *p, const char *fmt, ...);
/**
 * Trigger a warning over a lexer.
 */
CHECK_PRINTF(2, 3) void LexerWarning(Lex *p, const char *fmt, ...);

/// Test whether a lexer reached the end.
FORCE_INLINE Boolean IsLexerEndOfFile(Lex *p)
{
	return (p->pos >= p->lim || *p->pos == '\0') && !p->hasBufferedToken;
}

/// Test whether a lexer encountered an error.
FORCE_INLINE Boolean HasLexerError(Lex *p)
{
	return p->hasError;
}

/**
 * \brief Read and return next token.
 *
 * \param [in,out] p    A lexer, must not be `NULL`
 * \param [out]    dest Storage for the returned token, must not be `NULL`
 *
 * \return If a new token has been read, then `tok->text` is returned,
 *         `NULL` is returned if a parsing error has been encountered,
 *         or no more tokens are available.
 */
char *Lex_ReadToken(Lex *p, Tok *dest);
/**
 * \brief Read and return next token in the same line.
 *
 * This is a variant of `Lex_ReadToken()` useful to implement
 * a C Preprocessor, it avoids parsing spanning more than one line.
 * `\` followed by a newline is recognized and treated as a regular
 * space.
 */
char *Lex_ReadTokenOnLine(Lex *p, Tok *dest);
/**
 * \brief Expects an integral token, reading and returning its value.
 *
 * \param [in,out] p            A lexer, must not be `NULL`
 * \param [in]     optionalSign Allow an optional `+` or `-` sign before the
 *                              token, if set to `FALSE` only unsigned
 *                              integers are allowed.
 *
 * \return The token value, 0 on error, use `HasLexerError()` to distinguish
 *         between actual 0 and error value.
 */
long long Lex_ParseInt(Lex *p, Boolean optionalSign);
/**
 * \brief Expects a boolean token, reading and returning its value.
 *
 * \param [in,out] p            A lexer, must not be `NULL`
 * \param [in]     allowNumeric Convert numeric values to booleans, 0 for
 *                              `FALSE`, any other numeric value for `TRUE`
 *
 * \return The boolean value, `FALSE` on error or end of file,
 *         use `HasLexerError()` or `IsLexerEndOfFile()` to distinguish
 *         between actual `FALSE` and error value.
 */
Boolean Lex_ParseBool(Lex *p, Boolean allowNumeric);
/**
 * \brief Expects a floating point token, reading and returning its value.
 *
 * \param [in,out] p            A lexer, must not be `NULL`
 * \param [in]     optionalSign Allow an optional `+` or `-` sign before the
 *                              token, if set to `FALSE` only non-negative
 *                              values are allowed.
 *
 * \return The float value, 0 on error or end of file,
 *         use `HasLexerError()` or `IsLexerEndOfFile()` to distinguish
 *         between actual 0 and error value.
 */
double Lex_ParseFloat(Lex *p, Boolean optionalSign);

/**
 * \brief Read a one dimensional matrix (vector of length `n`) from `p` into `dest`.
 *
 * Matrix format is:
 * ```
 * (x y z w ...)
 * ```
 *
 * \return `TRUE` on success, `FALSE` on error.
 */
Boolean Lex_ParseMatrix1(Lex *p, float *dest, size_t n);
/**
 * \brief `Lex_ParseMatrix1()` variant for two dimensional matrixes.
 *
 * Matrix format is:
 * ```
 * ((x0 y0 z0 w0 ...) (x1 y1 z1 w1 ...) ...)
 * ```
 */
Boolean Lex_ParseMatrix2(Lex *p, float *dest, size_t n, size_t m);
/// `Lex_ParseMatrix1()` variant for tridimensional matrixes.
Boolean Lex_ParseMatrix3(Lex *p, float *dest, size_t n, size_t m, size_t u);

/// Discard any buffered token and any in text token up to a new line.
void Lex_SkipLine(Lex *p);
/**
 * \brief Skip every token until `tok` is encountered.
 *
 * \param [in,out] p   A lexer, must not be `NULL`
 * \param [in]     tok Token to look for, must not be `NULL`
 *
 * \return `tok` on success, `NULL` on error or end of file.
 */
char *Lex_SkipUntil(Lex *p, const char *tok);
/**
 * \brief Expect and skip section enclosed within braces.
 *
 * Braced sections are enclosed by punctuation tokens of id `P_BRACEOPEN` and
 * `P_BRACECLOSE`.
 *
 * \param [in,out] p               A lexer, must not be `NULL`
 * \param [in]     parseFirstBrace Whether the function should expect the next
 *                                 token to be the first brace of the section
 *                                 (`TRUE`) or it should assume the first brace
 *                                 has already been parsed (`FALSE`).
 *
 * \return `TRUE` if section was skipped successfully, `FALSE` on error
 *         (either unbalanced braces or unexpected token).
 */
Boolean Lex_SkipBracedSection(Lex *p, Boolean parseFirstBrace);

/**
 * \brief Expect a token, matching and returning it, raises error on mismatch.
 *
 * \param [in,out] p   A lexer, must not be `NULL`
 * \param [in]     tok Token to be expected, must not be `NULL`
 *
 * \return On success `tok` is returned, on error `NULL`.
 */
char *Lex_MatchToken(Lex *p, const char *tok);
/**
 * \brief Expect any token, raises an error if none is found.
 *
 * \param [in,out] p    A lexer, must not be `NULL`
 * \param [out]    dest Storage for returned token, must not be `NULL`
 *
 * \return On success `tok->text` is returned, on error `NULL`.
 */
char *Lex_MatchAnyToken(Lex *p, Tok *dest);
/**
 * \brief Expect a token of a specific `type` and `subtype`, raise error on mismatch.
 *
 * \param [in,out] p       A lexer, must not be `NULL`
 * \param [out]    dest    Storage for returned token, must not be `NULL`
 * \param [in]     type    Token type to be expected
 * \param [in]     subtype Subtype mask for the expected token
 *
 * \return On success `tok->text`, `NULL` otherwise.
 */
char *Lex_MatchTokenType(Lex *p, Tok *dest, int type, unsigned subtype);

/**
 * Check whether next token matches `tok`.
 *
 * If token matches it is read from `p` and returned, as in `Lex_ReadToken()`,
 * otherwise `p` is left unaltered (except for parsing errors).
 */
char *Lex_CheckToken(Lex *p, const char *tok);
/// Similar to `Lex_CheckToken()`, but matches by token `type` and `subtype`.
char *Lex_CheckTokenType(Lex *, Tok *dest, int type, unsigned subtype);

/**
 * \brief Peek next token from `p` and test whether it matches with `tok`.
 *
 * In no case next token is consumed from `p`, lexer is left unaltered
 * (except for parsing errors).
 */
char *Lex_PeekToken(Lex *p, const char *tok);
/// Similar to `Lex_PeekToken()`, but matches by token `type` and `subtype`.
char *Lex_PeekTokenType(Lex *p, Tok *dest, int type, unsigned subtype);

/**
 * \brief Place a token back into the lexer.
 *
 * Only one token may be placed back into the lexer at a time,
 * it will be returned back on the next call to `Lex_ReadToken()`.
 */
void Lex_UngetToken(Lex *p, const Tok *tok);

#endif