shlex 实现笔记

const StaticCharSet = struct {
    bitset: std.bit_set.StaticBitSet(256) = std.bit_set.StaticBitSet(256).initEmpty(),

    const Self = @This();

    fn init(comptime chars: []const u8) Self {
        var set = Self{};
        inline for (chars) |char| {
            set.add(char);
        }
        return set;
    }

    fn add(self: *Self, comptime char: u8) void {
        self.bitset.set(char);
    }

    fn contains(self: Self, char: u8) bool {
        return self.bitset.isSet(char);
    }
    
	// ...

    fn subtract(self: *Self, comptime other: Self) void {
        var tmp = other.bitset;
        tmp.toggleAll();
        self.bitset.setIntersection(tmp);
    }
};

const CharSets = struct {
    wordchars: StaticCharSet,        // 词汇字符: a-z, A-Z, 0-9, _
    punctuation_chars: StaticCharSet, // 标点符号: ();<>|&
    whitespace: StaticCharSet,       // 空白字符: space, tab, \r, \n
    commenters: StaticCharSet,       // 注释字符: #
    quotes: StaticCharSet,           // 引号字符: ', "
    escape: StaticCharSet,           // 转义字符: \
    escapedquotes: StaticCharSet,    // 可被转义的引号: "
};

const TokenState = enum(u8) {
    /// 等待有效输入，跳过空白/注释
    initial,
    /// 构建常规词汇 token，处理引号嵌入
    word,
    /// 聚合连续标点符号为单一 token
    punctuation,
    /// 单引号内容，POSIX 下支持有限转义
    single_quoted,
    /// 双引号内容，完整转义支持
    double_quoted,
    /// 转义序列处理，需维护前置状态
    escape,
    /// 输入结束，清理并完成解析
    eof,
};

// 字符级 pushback：处理标点符号边界
pushback_chars: ArrayList(u8),

fn pushChar(self: *Self, char: u8) !void {
    try self.pushback_chars.insert(0, char);  // LIFO 队列
}

fn getNextChar(self: *Self) ?u8 {
    if (!char_sets.punctuation_chars.isEmpty() and 
        self.pushback_chars.items.len > 0) {
        return self.popChar();
    }
    return self.readChar();  // 正常读取
}

fn handleWordState(self: *Self, char: ?u8) !?TokenAction {
    if (char == null) {
        self.lexer_state.state = .eof;
        return .finish_token;
    }

    const c = char.?;

    // 遇到空白字符，结束当前词汇
    if (char_sets.whitespace.contains(c)) {
        self.lexer_state.state = .initial;
        return .finish_token;
    }

    // POSIX 模式下可以在词汇中嵌入引号
    if (options.posix and c == '\'') {
        self.lexer_state.state = .single_quoted;
        return .continue_parsing;
    }

    // 继续添加字符到当前词汇
    if (char_sets.wordchars.contains(c)) {
        try self.token_buffer.append(c);
        return .continue_parsing;
    }

    // 遇到标点符号，需要回退处理
    if (!char_sets.punctuation_chars.isEmpty()) {
        try self.pushChar(c);  // 回退字符
    }

    self.lexer_state.state = .initial;
    return .finish_token;
}

fn handleQuotedState(self: *Self, char: ?u8, quote_char: u8) !?TokenAction {
    if (char == null) {
        return ShlexError.NoClosingQuotation;  // 未闭合的引号
    }

    const c = char.?;

    // 遇到闭合引号
    if (c == quote_char) {
        if (!options.posix) {
            // 非 POSIX 模式：引号本身也是 token 的一部分
            try self.token_buffer.append(c);
            self.lexer_state.state = .initial;
            return .finish_token;
        } else {
            // POSIX 模式：引号不是 token 的一部分
            self.lexer_state.state = .word;
            return .continue_parsing;
        }
    }

    // 双引号内可以处理转义字符
    if (options.posix and char_sets.escape.contains(c) and quote_char == '"') {
        self.lexer_state.escaped_state = self.lexer_state.state;
        self.lexer_state.state = .escape;
        return .continue_parsing;
    }

    // 普通字符直接添加
    try self.token_buffer.append(c);
    return .continue_parsing;
}

fn handleEscapeState(self: *Self, char: ?u8) !?TokenAction {
    if (char == null) {
        return ShlexError.NoEscapedCharacter;  // 转义后没有字符
    }

    const c = char.?;

    // POSIX shell中，只有引号本身或转义字符可以在引号内被转义
    switch (self.lexer_state.escaped_state) {
        .single_quoted => {
            if (c != '\\' and c != '\'') {
                try self.token_buffer.append('\\');  // 保留反斜杠
            }
        },
        .double_quoted => {
            if (c != '\\' and c != '"') {
                try self.token_buffer.append('\\');  // 保留反斜杠
            }
        },
        else => {},  // 其他状态下不特殊处理
    }

    try self.token_buffer.append(c);  // 添加被转义的字符
    self.lexer_state.state = self.lexer_state.escaped_state;  // 恢复前置状态
    return .continue_parsing;
}