lex.go

package sqlparser

import (
	"github.com/viant/parsly"
	"github.com/viant/parsly/matcher"
	smatcher "github.com/viant/sqlparser/matcher"

	"github.com/viant/parsly/matcher/option"
)

// Token represents a token
type Token int

const (
	whitespaceCode int = iota
	parenthesesCode
	nextCode
	identifierCode
	dotCode
	starTokenCode
	nullTokenCode
	notOperator
	binaryOperator
	logicalOperator
	assignOperator
	intLiteral
	numericLiteral
	boolLiteral
	nullKeyword
	singleQuotedStringLiteral
	rawSingleQuotedStringLiteral
	doubleQuotedStringLiteral
	caseBlock
	betweenToken
	orderDirection
	commentBlock
	selectKeyword
	updateKeyword
	setKeyword
	insertIntoKeyword
	insertValuesKeyword
	placeholderTokenCode
	selectorTokenCode
	tableTokenCode
	asKeyword
	selectionKindCode
	exceptKeyword
	selectionKind
	fromKeyword
	onKeyword
	joinToken
	whereKeyword
	groupByKeyword
	havingKeyword
	orderByKeyword
	onDuplicateKeyUpdate
	rangeOperator
	windowTokenCode
	literalCode
	exprToken
	keyTokenCode
	notNullToken
	createTableToken
	createToken
	indexToken
	defaultToken
	ifNotExistsToken
	ifExistsToken
	dropTableToken
	dropIndexToken
	deleteCode
	withKeyword
	recursiveKeyword
	unionKeyword
	globalKeyword
	registerKeyword
	typeKeyword
	ttlKeyword
	truncateTableKeyword
	collateKeyword
	whenKeyword
	thenKeyword
	elseKeyword
	endKeyword
	intervalKeyword
	intervalUnit
	limitKeyword
	nullTreatmentKeyword
	nullsKeyword
	separatorKeyword
	notLikeOperator
)

var whitespaceMatcher = parsly.NewToken(whitespaceCode, "whitespace", matcher.NewWhiteSpace())
var parenthesesMatcher = parsly.NewToken(parenthesesCode, "()", matcher.NewBlock('(', ')', '\\'))
var nextMatcher = parsly.NewToken(nextCode, ",", matcher.NewByte(','))
var asKeywordMatcher = parsly.NewToken(asKeyword, "AS", matcher.NewKeyword("as", &option.Case{}))
var starTokenMatcher = parsly.NewToken(starTokenCode, "*", matcher.NewByte('*'))
var notOperatorMatcher = parsly.NewToken(notOperator, "NOT", matcher.NewKeyword("not", &option.Case{}))
var bitwiseNotMatcher = parsly.NewToken(notOperator, "~", matcher.NewByte('~'))
var nullMatcher = parsly.NewToken(nullTokenCode, "NULL", matcher.NewKeyword("null", &option.Case{}))
var selectionKindMatcher = parsly.NewToken(selectionKindCode, "ALL|DISTINCT|STRUCT", selectionModifier{
	keywords:          matcher.NewSet([]string{"ALL", "DISTINCT", "STRUCT"}, &option.Case{}),
	selector:          smatcher.NewSelector(false),
	excludeStructCall: true,
})
var selectAsKeywordMatcher = parsly.NewToken(asKeyword, "AS", selectionModifier{
	keywords: matcher.NewKeyword("AS", &option.Case{}),
	selector: smatcher.NewSelector(false),
})
var selectStructKeywordMatcher = parsly.NewToken(selectionKindCode, "STRUCT", selectionModifier{
	keywords: matcher.NewKeyword("STRUCT", &option.Case{}),
	selector: smatcher.NewSelector(false),
})
var orderDirectionMatcher = parsly.NewToken(orderDirection, "ASC|DESC", matcher.NewSet([]string{
	"ASC", "DESC",
}, &option.Case{}))

// CASE delimiters must not consume identifier prefixes such as end_date.
var caseBlockMatcher = parsly.NewToken(caseBlock, "CASE", selectionModifier{
	keywords: matcher.NewKeyword("case", &option.Case{}),
	selector: aliasIdentifier{},
})
var whenKeywordMatcher = parsly.NewToken(whenKeyword, "WHEN", selectionModifier{
	keywords: matcher.NewKeyword("when", &option.Case{}),
	selector: aliasIdentifier{},
})
var thenKeywordMatcher = parsly.NewToken(thenKeyword, "THEN", selectionModifier{
	keywords: matcher.NewKeyword("then", &option.Case{}),
	selector: aliasIdentifier{},
})
var elseKeywordMatcher = parsly.NewToken(elseKeyword, "ELSE", selectionModifier{
	keywords: matcher.NewKeyword("else", &option.Case{}),
	selector: aliasIdentifier{},
})
var endKeywordMatcher = parsly.NewToken(endKeyword, "END", selectionModifier{
	keywords: matcher.NewKeyword("end", &option.Case{}),
	selector: aliasIdentifier{},
})
var intervalKeywordMatcher = parsly.NewToken(intervalKeyword, "INTERVAL", matcher.NewKeyword("interval", &option.Case{}))
var intervalUnitMatcher = parsly.NewToken(intervalUnit, "interval unit", matcher.NewSet([]string{"microsecond", "millisecond", "second", "minute", "hour", "day", "week", "month", "quarter", "year", "year_month", "day_hour", "day_minute", "day_second", "hour_minute", "hour_second", "minute_second", "day_microsecond", "hour_microsecond", "minute_microsecond", "second_microsecond"}, &option.Case{}))
var limitKeywordMatcher = parsly.NewToken(limitKeyword, "LIMIT", matcher.NewKeyword("limit", &option.Case{}))
var separatorKeywordMatcher = parsly.NewToken(separatorKeyword, "SEPARATOR", selectionModifier{
	keywords: matcher.NewKeyword("separator", &option.Case{}),
	selector: smatcher.NewSelector(false),
})
var ignoreKeywordMatcher = parsly.NewToken(nullTreatmentKeyword, "IGNORE", matcher.NewKeyword("ignore", &option.Case{}))
var respectKeywordMatcher = parsly.NewToken(nullTreatmentKeyword, "RESPECT", matcher.NewKeyword("respect", &option.Case{}))
var nullsKeywordMatcher = parsly.NewToken(nullsKeyword, "NULLS", matcher.NewKeyword("nulls", &option.Case{}))
var commentBlockMatcher = parsly.NewToken(commentBlock, "/* */", matcher.NewSeqBlock("/*", "*/"))
var inlineCommentMatcher = parsly.NewToken(commentBlock, "--", matcher.NewSeqBlock("--", "\n"))

var selectKeywordMatcher = parsly.NewToken(selectKeyword, "SELECT", matcher.NewKeyword("select", &option.Case{}))
var exceptKeywordMatcher = parsly.NewToken(exceptKeyword, "EXCEPT", matcher.NewKeyword("except", &option.Case{}))
var betweenKeywordMatcher = parsly.NewToken(betweenToken, "BETWEEN", matcher.NewKeyword("between", &option.Case{}))
var truncateKeywordMatcher = parsly.NewToken(truncateTableKeyword, "TRUNCATE", matcher.NewSpacedFragment("truncate table", &option.Case{}))

var fromKeywordMatcher = parsly.NewToken(fromKeyword, "FROM", matcher.NewKeyword("from", &option.Case{}))
var joinMatcher = parsly.NewToken(joinToken, "LEFT OUTER JOIN|LEFT JOIN|JOIN", matcher.NewSpacedSet([]string{
	"left outer join",
	"cross join",
	"left join",
	"inner join",
	"join",
}, &option.Case{}))

var keyMatcher = parsly.NewToken(keyTokenCode, "[RANGE|HASH|PRIMARY] KEY", matcher.NewSpacedSet([]string{
	"range key",
	"hash key",
	"primary key",
}, &option.Case{}))

var onKeywordMatcher = parsly.NewToken(onKeyword, "ON", matcher.NewKeyword("on", &option.Case{}))
var collateKeywordMatcher = parsly.NewToken(collateKeyword, "COLLATE", matcher.NewKeyword("collate", &option.Case{}))

var whereKeywordMatcher = parsly.NewToken(whereKeyword, "WHERE", matcher.NewKeyword("where", &option.Case{}))
var groupByMatcher = parsly.NewToken(groupByKeyword, "GROUP BY", matcher.NewSpacedFragment("group by", &option.Case{}))
var havingKeywordMatcher = parsly.NewToken(havingKeyword, "HAVING", matcher.NewKeyword("having", &option.Case{}))

var orderByKeywordMatcher = parsly.NewToken(orderByKeyword, "ORDER BY", matcher.NewSpacedFragment("order by", &option.Case{}))
var onDuplicateKeyUpdateMatcher = parsly.NewToken(onDuplicateKeyUpdate, "ON DUPLICATE KEY UPDATE", matcher.NewSpacedFragment("on duplicate key update", &option.Case{}))

var windowMatcher = parsly.NewToken(windowTokenCode, "LIMIT|OFFSET", matcher.NewSet([]string{"limit", "offset"}, &option.Case{}))

var updateKeywordMatcher = parsly.NewToken(updateKeyword, "UPDATE", matcher.NewKeyword("update", &option.Case{}))
var setKeywordMatcher = parsly.NewToken(setKeyword, "SET", matcher.NewKeyword("set", &option.Case{}))

var insertIntoKeywordMatcher = parsly.NewToken(insertIntoKeyword, "INSERT INTO", matcher.NewSpacedSet([]string{
	"insert into"}, &option.Case{}))

var insertValesKeywordMatcher = parsly.NewToken(insertValuesKeyword, "VALUES", matcher.NewKeyword("values", &option.Case{}))

var binaryOperatorMatcher = parsly.NewToken(binaryOperator, "binary OPERATOR", matcher.NewSpacedSet([]string{"+", "!=", "<>", ">=", "<=", "<<", ">>", "=", "-", ">", "<", "*", "/", "&", "^", "|", "in", "not in", "is not", "is", "like"}, &option.Case{}))
var notLikeOperatorMatcher = parsly.NewToken(notLikeOperator, "NOT LIKE", keywordSequence{"NOT", "LIKE"})
var assignOperatorMatcher = parsly.NewToken(assignOperator, "assign OPERATOR", matcher.NewSpacedSet([]string{"="}, &option.Case{}))

var logicalOperatorMatcher = parsly.NewToken(logicalOperator, "AND|OR", selectionModifier{
	keywords: matcher.NewSet([]string{"and", "or"}, &option.Case{}),
	selector: smatcher.NewSelector(false),
})
var rangeOperatorMatcher = parsly.NewToken(rangeOperator, ".. AND .. ", matcher.NewSet([]string{"and"}, &option.Case{}))

var nullKeywordMatcher = parsly.NewToken(nullKeyword, "NULL", matcher.NewKeyword("null", &option.Case{}))
var boolLiteralMatcher = parsly.NewToken(boolLiteral, "true|false", matcher.NewSet([]string{"true", "false"}, &option.Case{}))
var singleQuotedStringLiteralMatcher = parsly.NewToken(singleQuotedStringLiteral, `'...'`, smatcher.NewStringMatcher('\''))
var rawSingleQuotedStringLiteralMatcher = parsly.NewToken(rawSingleQuotedStringLiteral, `r'...'`, smatcher.NewPrefixedQuote([]byte{'r', 'R'}, '\'', '\\'))
var doubleQuotedStringLiteralMatcher = parsly.NewToken(doubleQuotedStringLiteral, `"..."`, smatcher.NewStringMatcher('"'))
var intLiteralMatcher = parsly.NewToken(intLiteral, `INT`, smatcher.NewIntMatcher())
var numericLiteralMatcher = parsly.NewToken(numericLiteral, `NUMERIC`, matcher.NewNumber())

var dotMatcher = parsly.NewToken(dotCode, ".", matcher.NewByte('.'))

var aliasIdentifierMatcher = parsly.NewToken(identifierCode, "ALIAS", aliasIdentifier{})

var identifierMatcher = parsly.NewToken(identifierCode, "IDENT", smatcher.NewIdentifier())
var selectorMatcher = parsly.NewToken(selectorTokenCode, "SELECTOR", smatcher.NewSelector(false))
var tableMatcher = parsly.NewToken(tableTokenCode, "TABLE MATCHER", smatcher.NewSelector(true))
var bracedTableMatcher = parsly.NewToken(tableTokenCode, "BRACED TABLE", smatcher.NewBracedTableSelector())

var placeholderMatcher = parsly.NewToken(placeholderTokenCode, "SELECTOR", smatcher.NewPlaceholder())
var literalMatcher = parsly.NewToken(literalCode, "LITERAL", matcher.NewNop())

var withKeywordMatcher = parsly.NewToken(withKeyword, "WITH", matcher.NewKeyword("with", &option.Case{}))
var recursiveKeywordMatcher = parsly.NewToken(recursiveKeyword, "RECURSIVE", matcher.NewKeyword("recursive", &option.Case{}))
var unionMatcher = parsly.NewToken(unionKeyword, "UNION|UNION ALL", matcher.NewSpacedSet([]string{
	"union all",
	"union",
}, &option.Case{}))
var exprMatcher = parsly.NewToken(exprToken, ",EXPR", matcher.NewNop())

var deleteMatcher = parsly.NewToken(deleteCode, "DELETE", matcher.NewKeyword("delete", &option.Case{}))
var notNullMatcher = parsly.NewToken(notNullToken, "NOT NULL", matcher.NewSpacedSet([]string{
	"not null"}, &option.Case{}))
var ifNotExistsMatcher = parsly.NewToken(ifNotExistsToken, "IF NOT EXISTS", matcher.NewSpacedSet([]string{
	"if not exists"}, &option.Case{}))

var globalMatcher = parsly.NewToken(globalKeyword, "GLOBAL", matcher.NewSpacedSet([]string{
	"global"}, &option.Case{}))

var ifExistsMatcher = parsly.NewToken(ifExistsToken, "IF EXISTS", matcher.NewSpacedSet([]string{
	"if exists"}, &option.Case{}))

var createTableMatcher = parsly.NewToken(createTableToken, "CREATE TABLE", matcher.NewSpacedSet([]string{
	"create table"}, &option.Case{}))

var createMatcher = parsly.NewToken(createToken, "CREATE", matcher.NewSpacedSet([]string{
	"create"}, &option.Case{}))

var indexMatcher = parsly.NewToken(indexToken, "INDEX", matcher.NewSpacedSet([]string{
	"index"}, &option.Case{}))

var defaultMatcher = parsly.NewToken(defaultToken, "DEFAULT", matcher.NewKeyword("default", &option.Case{}))

var dropTableMatcher = parsly.NewToken(dropTableToken, "DROP TABLE", matcher.NewSpacedSet([]string{
	"drop table"}, &option.Case{}))

var dropIndexMatcher = parsly.NewToken(dropIndexToken, "DROP INDEX", matcher.NewSpacedSet([]string{
	"drop index"}, &option.Case{}))

var registerKeywordMatcher = parsly.NewToken(registerKeyword, "REGISTER", matcher.NewKeyword("register", &option.Case{}))

var typeKeywordMatcher = parsly.NewToken(typeKeyword, "TYPE", matcher.NewKeyword("type", &option.Case{}))

var ttlKeywordMatcher = parsly.NewToken(ttlKeyword, "With TTL", matcher.NewFragment("with ttl", &option.Case{Sensitive: false}))