Commit b64a0a3b authored by Joel K. Pettersson's avatar Joel K. Pettersson
Browse files

Scanner & parser clean-up. Macro renaming.

Redesign scanner:
 * Add whitespace filtering level setting.
   Setting the level replaces the whitespace filtering
   functions in the table for the scanner instance.
 * Remove "nospace" versions of get/try functions.
   Simplify get function. Remove old space-skipping
   inline functions.

Use the new way of filtering whitespace in parser.
Also refactor parser scan_time() and its uses.

Further testing of line and char position numbering
needed, but now better than it was in v0.3.0.

Change naming of keyword-like macros to sauCamelCaseName:
 * Rename compiler attribute SAU__* macros.
 * Rename ArrType SAU_DEF_* macros.

Increase version to v0.3.1, with suffix 'c'.

Change log:
 * 2019-07-07: scanner: Add ws filter level setting.
       Remove nospace versions of functions.
       Use next design. Consistent for filtering,
       even if changed. Clear up ws types naming,
       really fill in everything. Tidying.
 * 2019-07-09: Scanner flag fix.
       (Recommitted after fixing the fix.)
       Set version to v0.3.1.
 * 2019-07-11: Microflensing.
       scanner: Remove "RAW" (NULL-pointer) ws level,
       rename "PASS" to "ALL" and make it the default
       (instead of "RED1").
 * 2019-07-29: parser: refactor scan_time() and its uses.
 * 2019-08-05: Place after new v0.3.0,
       with July syntax changes split out into v0.2.13.
 * 2019-08-06: FreeBSD portability fix:
       Fix parsing when -ffast-math breaks NAN checks.
       (Only build some files with -ffast-math.)
       Increase version suffix to 'b'.
       Move Makefile change down to v0.2.13.
 * 2019-08-15: Naming: SAU__* -> sau* (and make "*" CamelCase)
       mixer: add missing sauMalloclike
       Naming: SAU_DEF_* -> sau* (arrtype)
 * 2019-08-16: Re-order ramp run & skip function params.
 * 2019-08-17: Remove scanner ws levels between ALL and NONE.
       (Not needed in current program.)
       Increase version suffix to 'c'.
 * 2019-08-23: Add sauNoinline to scan_time().
parent 1631a7bb
......@@ -24,9 +24,9 @@
/**
* Declare array type using \p Name, with \p ElementType.
*
* Only declares type, not methods. See SAU_DEF_ArrType().
* Only declares type, not methods. See sauArrType().
*/
#define SAU_DEF_ArrType_TYPE(Name, ElementType) \
#define sauArrTypeStruct(Name, ElementType) \
typedef struct Name { \
ElementType *a; \
size_t count; \
......@@ -36,26 +36,26 @@ typedef struct Name { \
/**
* Declare array methods for \p Name, with \p ElementType.
*
* Only declares methods, not type. See SAU_DEF_ArrType().
* Only declares methods, not type. See sauArrType().
*
* The Name_*() methods defined are inline wrappers around the
* generic methods. If not blank, \p MethodPrefix will be used
* to prefix their names.
*/
#define SAU_DEF_ArrType_METHODS(Name, ElementType, MethodPrefix) \
static inline bool SAU__maybe_unused \
#define sauArrTypeMethods(Name, ElementType, MethodPrefix) \
static inline bool sauMaybeUnused \
MethodPrefix##Name##_add(Name *restrict o, const ElementType *restrict item) { \
return SAU_ArrType_add(o, item, sizeof(ElementType)); \
} \
static inline bool SAU__maybe_unused \
static inline bool sauMaybeUnused \
MethodPrefix##Name##_upsize(Name *restrict o, size_t count) { \
return SAU_ArrType_upsize(o, count, sizeof(ElementType)); \
} \
static inline void SAU__maybe_unused \
static inline void sauMaybeUnused \
MethodPrefix##Name##_clear(Name *restrict o) { \
SAU_ArrType_clear(o); \
} \
static inline bool SAU__maybe_unused \
static inline bool sauMaybeUnused \
MethodPrefix##Name##_memdup(Name *restrict o, \
const ElementType **restrict dst) { \
return SAU_ArrType_memdup(o, (const void**) dst, sizeof(ElementType)); \
......@@ -64,15 +64,15 @@ MethodPrefix##Name##_memdup(Name *restrict o, \
/**
* Declare both type and methods for \p Name, with \p ElementType.
*
* Combines SAU_DEF_ArrType_TYPE() and SAU_DEF_ArrType_METHODS().
* Combines sauArrTypeStruct() and sauArrTypeMethods().
*
* The Name_*() methods defined are inline wrappers around the
* generic methods. If not blank, \p MethodPrefix will be used
* to prefix their names.
*/
#define SAU_DEF_ArrType(Name, ElementType, MethodPrefix) \
SAU_DEF_ArrType_TYPE(Name, ElementType) \
SAU_DEF_ArrType_METHODS(Name, ElementType, MethodPrefix)
#define sauArrType(Name, ElementType, MethodPrefix) \
sauArrTypeStruct(Name, ElementType) \
sauArrTypeMethods(Name, ElementType, MethodPrefix)
bool SAU_ArrType_add(void *restrict o,
const void *restrict item, size_t item_size);
......@@ -83,4 +83,4 @@ bool SAU_ArrType_memdup(void *restrict o,
const void **restrict dst, size_t item_size);
/** Byte (uint8_t) array type. */
SAU_DEF_ArrType(SAU_ByteArr, uint8_t, );
sauArrType(SAU_ByteArr, uint8_t, );
......@@ -18,7 +18,7 @@ struct SAU_AudioDev;
typedef struct SAU_AudioDev SAU_AudioDev;
SAU_AudioDev *SAU_open_AudioDev(uint16_t channels, uint32_t *restrict srate)
SAU__malloclike;
sauMalloclike;
void SAU_close_AudioDev(SAU_AudioDev *restrict o);
uint32_t SAU_AudioDev_get_srate(const SAU_AudioDev *restrict o);
......
......@@ -170,8 +170,8 @@ struct SAU_File {
uint8_t buf[SAU_FILE_BUFSIZ];
};
SAU_File *SAU_create_File(void) SAU__malloclike;
SAU_File *SAU_create_sub_File(SAU_File *restrict parent) SAU__malloclike;
SAU_File *SAU_create_File(void) sauMalloclike;
SAU_File *SAU_create_sub_File(SAU_File *restrict parent) sauMalloclike;
SAU_File *SAU_destroy_File(SAU_File *restrict o);
bool SAU_File_fopenrb(SAU_File *restrict o, const char *restrict path);
......
......@@ -66,6 +66,7 @@ SAU_Lexer *SAU_create_Lexer(SAU_SymTab *restrict symtab) {
#if SAU_LEXER_QUIET
o->sc->s_flags |= SAU_SCAN_S_QUIET;
#endif
SAU_Scanner_setws_level(o->sc, SAU_SCAN_WS_NONE);
return o;
ERROR:
SAU_destroy_Lexer(o);
......@@ -102,14 +103,14 @@ void SAU_Lexer_close(SAU_Lexer *restrict o) {
SAU_Scanner_close(o->sc);
}
static void handle_invalid(SAU_Lexer *o, uint8_t c SAU__maybe_unused) {
static void handle_invalid(SAU_Lexer *o, uint8_t c sauMaybeUnused) {
SAU_ScriptToken *t = &o->token;
t->type = SAU_T_INVALID;
t->data.b = 0;
}
static void handle_eof(SAU_Lexer *restrict o,
uint8_t c SAU__maybe_unused) {
uint8_t c sauMaybeUnused) {
SAU_Scanner *sc = o->sc;
SAU_ScriptToken *t = &o->token;
t->type = SAU_T_INVALID;
......@@ -125,7 +126,7 @@ static void handle_special(SAU_Lexer *restrict o, uint8_t c) {
}
static void handle_numeric_value(SAU_Lexer *restrict o,
uint8_t c SAU__maybe_unused) {
uint8_t c sauMaybeUnused) {
SAU_Scanner *sc = o->sc;
SAU_ScriptToken *t = &o->token;
double d;
......@@ -137,7 +138,7 @@ static void handle_numeric_value(SAU_Lexer *restrict o,
}
static void handle_identifier(SAU_Lexer *restrict o,
uint8_t c SAU__maybe_unused) {
uint8_t c sauMaybeUnused) {
SAU_Scanner *sc = o->sc;
SAU_ScriptToken *t = &o->token;
const void *str;
......@@ -162,7 +163,7 @@ bool SAU_Lexer_get(SAU_Lexer *restrict o, SAU_ScriptToken *restrict t) {
SAU_Scanner *sc = o->sc;
uint8_t c;
REGET:
c = SAU_Scanner_getc_nospace(sc);
c = SAU_Scanner_getc(sc);
switch (c) {
case 0x00:
handle_eof(o, c);
......@@ -304,7 +305,7 @@ bool SAU_Lexer_get_special(SAU_Lexer *restrict o,
SAU_Scanner *sc = o->sc;
uint8_t c;
for (;;) {
c = SAU_Scanner_getc_nospace(sc);
c = SAU_Scanner_getc(sc);
if (c == 0) {
handle_eof(o, c);
break;
......
......@@ -39,7 +39,7 @@ typedef struct SAU_ScriptToken {
struct SAU_Lexer;
typedef struct SAU_Lexer SAU_Lexer;
SAU_Lexer *SAU_create_Lexer(SAU_SymTab *restrict symtab) SAU__malloclike;
SAU_Lexer *SAU_create_Lexer(SAU_SymTab *restrict symtab) sauMalloclike;
void SAU_destroy_Lexer(SAU_Lexer *restrict o);
bool SAU_Lexer_open(SAU_Lexer *restrict o,
......
......@@ -91,7 +91,7 @@ typedef struct VAState {
uint32_t duration_ms;
} VAState;
SAU_DEF_ArrType(VoAlloc, VAState, _)
sauArrType(VoAlloc, VAState, _)
/*
* Returns the longest operator duration among top-level operators for
......@@ -189,7 +189,7 @@ typedef struct OAState {
//uint32_t duration_ms;
} OAState;
SAU_DEF_ArrType(OpAlloc, OAState, _)
sauArrType(OpAlloc, OAState, _)
/*
* Get operator ID for event, setting it to \p op_id.
......@@ -251,8 +251,8 @@ static void OpAlloc_clear(OpAlloc *restrict o) {
_OpAlloc_clear(o);
}
SAU_DEF_ArrType(OpRefArr, SAU_ProgramOpRef, )
SAU_DEF_ArrType(OpDataArr, SAU_ProgramOpData, )
sauArrType(OpRefArr, SAU_ProgramOpRef, )
sauArrType(OpDataArr, SAU_ProgramOpData, )
typedef struct ParseConv {
SAU_PtrList ev_list;
......
......@@ -110,11 +110,10 @@ static double scan_num_r(NumParser *restrict o, uint8_t pri, uint32_t level) {
double num;
bool minus = false;
uint8_t c;
if (level > 0) SAU_Scanner_skipws(sc);
if (level == 1) SAU_Scanner_setws_level(sc, SAU_SCAN_WS_NONE);
c = SAU_Scanner_getc(sc);
if ((level > 0) && (c == '+' || c == '-')) {
if (c == '-') minus = true;
SAU_Scanner_skipws(sc);
c = SAU_Scanner_getc(sc);
}
if (c == '(') {
......@@ -122,9 +121,7 @@ static double scan_num_r(NumParser *restrict o, uint8_t pri, uint32_t level) {
if (minus) num = -num;
if (level == 0)
return num;
goto EVAL;
}
if (o->numsym_f && IS_ALPHA(c)) {
} else if (o->numsym_f && IS_ALPHA(c)) {
SAU_Scanner_ungetc(sc);
num = o->numsym_f(sc);
if (isnan(num))
......@@ -138,17 +135,12 @@ static double scan_num_r(NumParser *restrict o, uint8_t pri, uint32_t level) {
return NAN;
if (minus) num = -num;
}
EVAL:
if (pri == 0)
return num; /* defer all */
for (;;) {
if (isinf(num)) o->has_infnum = true;
if (level > 0) SAU_Scanner_skipws(sc);
c = SAU_Scanner_getc(sc);
switch (c) {
case SAU_SCAN_SPACE:
case SAU_SCAN_LNBRK:
break;
case '(':
num *= scan_num_r(o, 255, level+1);
break;
......@@ -185,10 +177,12 @@ DEFER:
SAU_Scanner_ungetc(sc);
return num;
}
static bool SAU__noinline scan_num(SAU_Scanner *restrict o,
static sauNoinline bool scan_num(SAU_Scanner *restrict o,
NumSym_f scan_numsym, float *restrict var) {
NumParser np = {o, scan_numsym, o->sf, false};
uint8_t ws_level = o->ws_level;
float num = scan_num_r(&np, 0, 0);
SAU_Scanner_setws_level(o, ws_level); // restore if changed
if (isnan(num))
return false;
if (isinf(num)) np.has_infnum = true;
......@@ -201,16 +195,19 @@ static bool SAU__noinline scan_num(SAU_Scanner *restrict o,
return true;
}
static bool scan_time(SAU_Scanner *restrict o, float *restrict var) {
static sauNoinline bool scan_time(SAU_Scanner *restrict o,
uint32_t *restrict var) {
SAU_ScanFrame sf = o->sf;
float num;
if (!scan_num(o, NULL, &num))
float num_s;
if (!scan_num(o, NULL, &num_s))
return false;
if (num < 0.f) {
if (num_s < 0.f) {
SAU_Scanner_warning(o, &sf, "discarding negative time value");
return false;
}
*var = num;
uint32_t num_ms;
num_ms = lrint(num_s * 1000.f);
*var = num_ms;
return true;
}
......@@ -380,8 +377,9 @@ static bool scan_ramp(SAU_Scanner *restrict o, NumSym_f scan_numsym,
time_ms = ramp->time_ms;
}
for (;;) {
uint8_t c = SAU_Scanner_getc_nospace(o);
uint8_t c = SAU_Scanner_getc(o);
switch (c) {
case SAU_SCAN_SPACE:
case SAU_SCAN_LNBRK:
break;
case 'c': {
......@@ -391,12 +389,9 @@ static bool scan_ramp(SAU_Scanner *restrict o, NumSym_f scan_numsym,
curve = type;
}
break; }
case 't': {
float time;
if (scan_time(o, &time)) {
time_ms = lrint(time * 1000.f);
}
break; }
case 't':
scan_time(o, &time_ms);
break;
case 'v':
if (scan_num(o, scan_numsym, &vt))
goal = true;
......@@ -527,10 +522,8 @@ static bool parse_waittime(ParseLevel *restrict pl) {
}
pl->last_event->ev_flags |= SAU_SDEV_ADD_WAIT_DURATION;
} else {
float wait;
uint32_t wait_ms;
if (scan_time(sc, &wait)) {
wait_ms = lrint(wait * 1000.f);
if (scan_time(sc, &wait_ms)) {
pl->next_wait_ms += wait_ms;
}
}
......@@ -900,8 +893,10 @@ static bool parse_settings(ParseLevel *restrict pl) {
SAU_Scanner *sc = o->sc;
pl->location = SDPL_IN_DEFAULTS;
for (;;) {
uint8_t c = SAU_Scanner_getc_nospace(sc);
uint8_t c = SAU_Scanner_getc(sc);
switch (c) {
case SAU_SCAN_SPACE:
break;
case 'a':
if (scan_num(sc, NULL, &sl->sopt.ampmult))
sl->sopt.changed |= SAU_SOPT_AMPMULT;
......@@ -926,19 +921,19 @@ static bool parse_settings(ParseLevel *restrict pl) {
if (scan_num(sc, NULL, &sl->sopt.def_relfreq))
sl->sopt.changed |= SAU_SOPT_DEF_RATIO;
break;
case 't': {
float time;
if (scan_time(sc, &time)) {
sl->sopt.def_time_ms = lrint(time * 1000.f);
case 't':
if (scan_time(sc, &sl->sopt.def_time_ms)) {
sl->sopt.changed |= SAU_SOPT_DEF_TIME;
}
break; }
break;
default:
SAU_Scanner_ungetc(sc);
return true; /* let parse_level() take care of it */
goto UNKNOWN;
}
}
return false;
UNKNOWN:
SAU_Scanner_ungetc(sc);
return true; /* let parse_level() take care of it */
}
static bool parse_level(SAU_Parser *restrict o, ParseLevel *restrict parent_pl,
......@@ -1027,8 +1022,10 @@ static bool parse_step(ParseLevel *restrict pl) {
SAU_ScriptOpData *op = pl->operator;
pl->location = SDPL_IN_EVENT;
for (;;) {
uint8_t c = SAU_Scanner_getc_nospace(sc);
uint8_t c = SAU_Scanner_getc(sc);
switch (c) {
case SAU_SCAN_SPACE:
break;
case 'P':
if ((pl->pl_flags & SDPL_NESTED_SCOPE) != 0)
goto UNKNOWN;
......@@ -1055,12 +1052,9 @@ static bool parse_step(ParseLevel *restrict pl) {
case 'r':
if (parse_ev_freq(pl, true)) goto UNKNOWN;
break;
case 's': {
float silence;
if (scan_time(sc, &silence)) {
op->silence_ms = lrint(silence * 1000.f);
}
break; }
case 's':
scan_time(sc, &op->silence_ms);
break;
case 't':
if (SAU_Scanner_tryc(sc, '*')) {
/* later fitted or kept at default value */
......@@ -1075,10 +1069,8 @@ static bool parse_step(ParseLevel *restrict pl) {
op->op_flags &= ~SAU_SDOP_TIME_DEFAULT;
op->time_ms = SAU_TIME_INF;
} else {
float time;
if (scan_time(sc, &time)) {
if (scan_time(sc, &op->time_ms)) {
op->op_flags &= ~SAU_SDOP_TIME_DEFAULT;
op->time_ms = lrint(time * 1000.f);
}
}
op->op_params |= SAU_POPP_TIME;
......@@ -1090,12 +1082,13 @@ static bool parse_step(ParseLevel *restrict pl) {
op->wave = wave;
break; }
default:
UNKNOWN:
SAU_Scanner_ungetc(sc);
return true; /* let parse_level() take care of it */
goto UNKNOWN;
}
}
return false;
UNKNOWN:
SAU_Scanner_ungetc(sc);
return true; /* let parse_level() take care of it */
}
enum {
......@@ -1114,8 +1107,10 @@ static bool parse_level(SAU_Parser *restrict o, ParseLevel *restrict parent_pl,
++o->call_level;
SAU_Scanner *sc = o->sc;
for (;;) {
uint8_t c = SAU_Scanner_getc_nospace(sc);
uint8_t c = SAU_Scanner_getc(sc);
switch (c) {
case SAU_SCAN_SPACE:
break;
case SAU_SCAN_LNBRK:
if (pl.scope == SCOPE_TOP) {
/*
......
......@@ -45,6 +45,7 @@ SAU_Scanner *SAU_create_Scanner(SAU_SymTab *restrict symtab) {
size_t filters_size = sizeof(SAU_ScanFilter_f) * SAU_SCAN_FILTER_COUNT;
o->filters = SAU_memdup(SAU_Scanner_def_filters, filters_size);
if (!o->filters) goto ERROR;
o->ws_level = SAU_SCAN_WS_ALL;
o->strbuf = calloc(1, STRBUF_LEN);
if (!o->strbuf) goto ERROR;
return o;
......@@ -111,7 +112,7 @@ void SAU_Scanner_close(SAU_Scanner *restrict o) {
/* Valid characters in identifiers. */
#define IS_SYMCHAR(c) (IS_ALNUM(c) || (c) == '_')
static uint8_t filter_symchar(SAU_File *restrict o SAU__maybe_unused,
static uint8_t filter_symchar(SAU_File *restrict o sauMaybeUnused,
uint8_t c) {
return IS_SYMCHAR(c) ? c : 0;
}
......@@ -164,52 +165,83 @@ uint8_t SAU_Scanner_filter_invalid(SAU_Scanner *restrict o, uint8_t c) {
return 0;
}
uint8_t status = SAU_File_STATUS(f);
if ((status & SAU_FILE_ERROR) != 0) {
if (status & SAU_FILE_ERROR) {
SAU_Scanner_error(o, NULL,
"file reading failed");
}
return SAU_SCAN_EOF;
}
static inline void pos_past_linebreak(SAU_Scanner *restrict o,
size_t char_num) {
++o->sf.line_num;
o->sf.char_num = char_num;
}
/**
* Get characters until the next is neither a space nor a tab.
* Return standard space marker (for space or tab).
*
* \return SAU_SCAN_SPACE
*/
uint8_t SAU_Scanner_filter_space(SAU_Scanner *restrict o,
uint8_t c SAU__maybe_unused) {
SAU_File *f = o->f;
o->sf.char_num += SAU_File_skipspace(f);
uint8_t SAU_Scanner_filter_space_keep(SAU_Scanner *restrict o sauMaybeUnused,
uint8_t c sauMaybeUnused) {
o->sf.c_flags |= SAU_SCAN_C_SPACE;
return SAU_SCAN_SPACE;
}
/**
* Get characters until the next is not a linebreak.
* Handle linebreak portably (move past CR for LF),
* and return standard linebreak marker.
*
* \return SAU_SCAN_LNBRK
*/
uint8_t SAU_Scanner_filter_linebreaks(SAU_Scanner *restrict o, uint8_t c) {
uint8_t SAU_Scanner_filter_linebreak_keep(SAU_Scanner *restrict o, uint8_t c) {
SAU_File *f = o->f;
if (c == '\n') SAU_File_TRYC(f, '\r');
while (SAU_File_trynewline(f)) {
++o->sf.line_num;
o->sf.char_num = 0;
}
o->sf.c_flags |= (SAU_SCAN_C_LNBRK | SAU_SCAN_C_LNBRK_POSUP);
return SAU_SCAN_LNBRK;
}
/**
* Get characters until the next character ends the line (or file).
* Skip spaces and/or linebreaks.
*
* \return 0
*/
uint8_t SAU_Scanner_filter_ws_none(SAU_Scanner *restrict o, uint8_t c) {
SAU_File *f = o->f;
if (c == '\n') {
SAU_File_TRYC(f, '\r');
} else if (c != '\r') {
o->sf.char_num += SAU_File_skipspace(f);
return 0;
}
o->sf.c_flags |= SAU_SCAN_C_LNBRK;
o->sf.c_flags &= ~SAU_SCAN_C_LNBRK_POSUP;
pos_past_linebreak(o, 0);
size_t space_count;
LNBRK:
while (SAU_File_trynewline(f)) pos_past_linebreak(o, 0);
space_count = SAU_File_skipspace(f);
if (space_count > 0) {
o->sf.char_num = space_count;
goto LNBRK;
}
return 0;
}
/**
* Skip characters until the next character ends the line (or file).
*
* Call for a character to use it as a line comment opener.
*
* \return SAU_SCAN_SPACE
* \return 0
*/
uint8_t SAU_Scanner_filter_linecomment(SAU_Scanner *restrict o,
uint8_t c SAU__maybe_unused) {
uint8_t c sauMaybeUnused) {
SAU_File *f = o->f;
o->sf.char_num += SAU_File_skipline(f);
return SAU_SCAN_SPACE;
return 0;
}
/**
......@@ -220,8 +252,11 @@ uint8_t SAU_Scanner_filter_linecomment(SAU_Scanner *restrict o,
*
* Does not set the linebreak flag. Linebreaks within a block comment
* are ignored (commented out), apart from in line numbering.
* A block comment counts syntactically as a single space (so that
* it cannot silently be placed between characters in a token),
* unless all whitespace is filtered out.
*
* \return SAU_SCAN_SPACE or SAU_SCAN_EOF (on unterminated comment)
* \return filtered SAU_SCAN_SPACE, or SAU_SCAN_EOF (on unterminated comment)
*/
uint8_t SAU_Scanner_filter_blockcomment(SAU_Scanner *restrict o,
uint8_t check_c) {
......@@ -254,7 +289,7 @@ uint8_t SAU_Scanner_filter_blockcomment(SAU_Scanner *restrict o,
}
o->sf.line_num = line_num;
o->sf.char_num = char_num;
return SAU_SCAN_SPACE;
return SAU_Scanner_usefilter(o, SAU_SCAN_SPACE, SAU_SCAN_SPACE);
}
/**
......@@ -264,7 +299,7 @@ uint8_t SAU_Scanner_filter_blockcomment(SAU_Scanner *restrict o,
* handling comment if present, otherwise simply returning the first
* character.
*
* \return \p c, SAU_SCAN_SPACE, or SAU_SCAN_EOF (on unterminated comment)
* \return \p c, 0, filtered SAU_SCAN_SPACE, or SAU_SCAN_EOF
*/
uint8_t SAU_Scanner_filter_slashcomments(SAU_Scanner *restrict o, uint8_t c) {
SAU_File *f = o->f;
......@@ -290,7 +325,7 @@ uint8_t SAU_Scanner_filter_slashcomments(SAU_Scanner *restrict o, uint8_t c) {
* for the first character position only. (For example,
* git-style comments, or old Fortran comments.)
*
* \return \p c or SAU_SCAN_SPACE
* \return \p c or 0
*/
uint8_t SAU_Scanner_filter_char1comments(SAU_Scanner *restrict o, uint8_t c) {
if (o->sf.char_num == 1)
......@@ -314,11 +349,11 @@ const SAU_ScanFilter_f SAU_Scanner_def_filters[SAU_SCAN_FILTER_COUNT] = {
/* ACK 0x06 */ SAU_Scanner_filter_invalid,
/* BEL '\a' */ SAU_Scanner_filter_invalid, // SAU_FILE_MARKER
/* BS '\b' */ SAU_Scanner_filter_invalid,
/* HT '\t' */ SAU_Scanner_filter_space,
/* LF '\n' */ SAU_Scanner_filter_linebreaks,
/* HT '\t' */ SAU_Scanner_filter_space_keep,
/* LF '\n' */ SAU_Scanner_filter_linebreak_keep,
/* VT '\v' */ SAU_Scanner_filter_invalid,
/* FF '\f' */ SAU_Scanner_filter_invalid,
/* CR '\r' */ SAU_Scanner_filter_linebreaks,
/* CR '\r' */ SAU_Scanner_filter_linebreak_keep,
/* SO 0x0E */ SAU_Scanner_filter_invalid,
/* SI 0x0F */ SAU_Scanner_filter_invalid,
/* DLE 0x10 */ SAU_Scanner_filter_invalid,
......@@ -337,7 +372,7 @@ const SAU_ScanFilter_f SAU_Scanner_def_filters[SAU_SCAN_FILTER_COUNT] = {
/* GS 0x1D */ SAU_Scanner_filter_invalid,
/* RS 0x1E */ SAU_Scanner_filter_invalid,
/* US 0x1F */ SAU_Scanner_filter_invalid,
/* ' ' */ SAU_Scanner_filter_space,
/* ' ' */ SAU_Scanner_filter_space_keep,
/* '!' */ NULL,
/* '"' */ NULL,
/* '#' */ SAU_Scanner_filter_linecomment,
......@@ -435,6 +470,31 @@ const SAU_ScanFilter_f SAU_Scanner_def_filters[SAU_SCAN_FILTER_COUNT] = {
/* DEL 0x7F */ SAU_Scanner_filter_invalid,
};
/**
* Set filter functions for whitespace characters to a standard set.
*
* \return old ws_level value
*/
uint8_t SAU_Scanner_setws_level(SAU_Scanner *restrict o, uint8_t ws_level) {
uint8_t old_level = o->ws_level;
switch (ws_level) {
case SAU_SCAN_WS_ALL: // default level
o->filters['\t'] = SAU_Scanner_filter_space_keep;
o->filters['\n'] = SAU_Scanner_filter_linebreak_keep;
o->filters['\r'] = SAU_Scanner_filter_linebreak_keep;
o->filters[' '] = SAU_Scanner_filter_space_keep;
break;
case SAU_SCAN_WS_NONE:
o->filters['\t'] = SAU_Scanner_filter_ws_none;
o->filters['\n'] = SAU_Scanner_filter_ws_none;
o->filters['\r'] = SAU_Scanner_filter_ws_none;
o->filters[' '] = SAU_Scanner_filter_ws_none;
break;
}
o->ws_level = ws_level;
return old_level;
}
/*
* Assign scan frame from undo buffer.
*/
......@@ -454,17 +514,17 @@ static void prepare_frame(SAU_Scanner *restrict o) {
restore_frame(o, --o->unget_num);
return;
}
if ((o->s_flags & SAU_SCAN_S_DISCARD) != 0) {
if (o->s_flags & SAU_SCAN_S_DISCARD) {
o->s_flags &= ~SAU_SCAN_S_DISCARD;
} else {
o->undo_pos = (o->undo_pos + 1) & SAU_SCAN_UNGET_MAX;