The following is the tokenizer, written in Flex (see [FLEX]) notation. The tokenizer is case-insensitive.
The two occurrences of "\377" represent the highest character number that current versions of Flex can deal with (decimal 255). They should be read as "\4177777" (decimal 1114111), which is the highest possible code point in Unicode/ISO-10646.
%option case-insensitive
h [0-9a-f]
nonascii [\200-\377]
unicode \\{h}{1,6}[ \t\r\n\f]?
escape {unicode}|\\[ -~\200-\377]
nmstart [a-z]|{nonascii}|{escape}
nmchar [a-z0-9-]|{nonascii}|{escape}
string1 \"([\t !#$%&(-~]|\\{nl}|\'|{nonascii}|{escape})*\"
string2 \'([\t !#$%&(-~]|\\{nl}|\"|{nonascii}|{escape})*\'
ident {nmstart}{nmchar}*
name {nmchar}+
num [0-9]+|[0-9]*"."[0-9]+
string {string1}|{string2}
url ([!#$%&*-~]|{nonascii}|{escape})*
w [ \t\r\n\f]*
nl \n|\r\n|\r|\f
range \?{1,6}|{h}(\?{0,5}|{h}(\?{0,4}|{h}(\?{0,3}|{h}(\?{0,2}|{h}(\??|{h})))))
%%
[ \t\r\n\f]+ {return S;}
\/\*[^*]*\*+([^/][^*]*\*+)*\/ /* ignore comments */
"" {return CDC;}
"~=" {return INCLUDES;}
"|=" {return DASHMATCH;}
{string} {return STRING;}
{ident} {return IDENT;}
"#"{name} {return HASH;}
"@import" {return IMPORT_SYM;}
"@page" {return PAGE_SYM;}
"@media" {return MEDIA_SYM;}
"@font-face" {return FONT_FACE_SYM;}
"@charset" {return CHARSET_SYM;}
"@"{ident} {return ATKEYWORD;}
"!{w}important" {return IMPORTANT_SYM;}
{num}em {return EMS;}
{num}ex {return EXS;}
{num}px {return LENGTH;}
{num}cm {return LENGTH;}
{num}mm {return LENGTH;}
{num}in {return LENGTH;}
{num}pt {return LENGTH;}
{num}pc {return LENGTH;}
{num}deg {return ANGLE;}
{num}rad {return ANGLE;}
{num}grad {return ANGLE;}
{num}ms {return TIME;}
{num}s {return TIME;}
{num}Hz {return FREQ;}
{num}kHz {return FREQ;}
{num}{ident} {return DIMEN;}
{num}% {return PERCENTAGE;}
{num} {return NUMBER;}
"url("{w}{string}{w}")" {return URI;}
"url("{w}{url}{w}")" {return URI;}
{ident}"(" {return FUNCTION;}
U\+{range} {return UNICODERANGE;}
U\+{h}{1,6}-{h}{1,6} {return UNICODERANGE;}
. {return *yytext;}
