hparser.c

/*
 * Copyright 1999-2016, Gisle Aas
 * Copyright 1999-2000, Michael A. Chase
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the same terms as Perl itself.
 */

#ifndef EXTERN
#define EXTERN extern
#endif

#include "hctype.h"    /* isH...() macros */
#include "tokenpos.h"  /* dTOKEN; PUSH_TOKEN() */


const static
struct literal_tag {
    int len;
    char* str;
    int is_cdata;
}
literal_mode_elem[] =
{
    {6, "script", 1},
    {5, "style", 1},
    {3, "xmp", 1},
    {6, "iframe", 1},
    {9, "plaintext", 1},
    {5, "title", 0},
    {8, "textarea", 0},
    {0, 0, 0}
};

enum argcode {
    ARG_SELF = 1,  /* need to avoid '\0' in argspec string */
    ARG_TOKENS,
    ARG_TOKENPOS,
    ARG_TOKEN0,
    ARG_TAGNAME,
    ARG_TAG,
    ARG_ATTR,
    ARG_ATTRARR,
    ARG_ATTRSEQ,
    ARG_TEXT,
    ARG_DTEXT,
    ARG_IS_CDATA,
    ARG_SKIPPED_TEXT,
    ARG_OFFSET,
    ARG_OFFSET_END,
    ARG_LENGTH,
    ARG_LINE,
    ARG_COLUMN,
    ARG_EVENT,
    ARG_UNDEF,
    ARG_LITERAL, /* Always keep last */

    /* extra flags always encoded first */
    ARG_FLAG_FLAT_ARRAY
};

static const char * const argname[] = {
    /* Must be in the same order as enum argcode */
    "self",     /* ARG_SELF */
    "tokens",   /* ARG_TOKENS */
    "tokenpos", /* ARG_TOKENPOS */
    "token0",   /* ARG_TOKEN0 */
    "tagname",  /* ARG_TAGNAME */
    "tag",      /* ARG_TAG */
    "attr",     /* ARG_ATTR */
    "@attr",    /* ARG_ATTRARR */
    "attrseq",  /* ARG_ATTRSEQ */
    "text",     /* ARG_TEXT */
    "dtext",    /* ARG_DTEXT */
    "is_cdata", /* ARG_IS_CDATA */
    "skipped_text", /* ARG_SKIPPED_TEXT */
    "offset",   /* ARG_OFFSET */
    "offset_end", /* ARG_OFFSET_END */
    "length",   /* ARG_LENGTH */
    "line",     /* ARG_LINE */
    "column",   /* ARG_COLUMN */
    "event",    /* ARG_EVENT */
    "undef",    /* ARG_UNDEF */
    /* ARG_LITERAL (not compared) */
    /* ARG_FLAG_FLAT_ARRAY */
};

#define CASE_SENSITIVE(p_state) \
         ((p_state)->xml_mode || (p_state)->case_sensitive)
#define STRICT_NAMES(p_state) \
         ((p_state)->xml_mode || (p_state)->strict_names)
#define ALLOW_EMPTY_TAG(p_state) \
         ((p_state)->xml_mode || (p_state)->empty_element_tags)

static void flush_pending_text(PSTATE* p_state, SV* self);

/*
 * Parser functions.
 *
 *   parse()                       - top level entry point.
 *                                   deals with text and calls one of its
 *                                   subordinate parse_*() routines after
 *                                   looking at the first char after "<"
 *     parse_decl()                - deals with declarations         <!...>
 *       parse_comment()           - deals with <!-- ... -->
 *       parse_marked_section      - deals with <![ ... [ ... ]]>
 *     parse_end()                 - deals with end tags             </...>
 *     parse_start()               - deals with start tags           <A...>
 *     parse_process()             - deals with process instructions <?...>
 *     parse_null()                - deals with anything else        <....>
 *
 *     report_event() - called whenever any of the parse*() routines
 *                      has recongnized something.
 */

static void
report_event(PSTATE* p_state,
	     event_id_t event,
	     char *beg, char *end, U32 utf8,
	     token_pos_t *tokens, int num_tokens,
	     SV* self
	    )
{
    struct p_handler *h;
    dTHX;
    dSP;
    AV *array;
    STRLEN my_na;
    char *argspec;
    char *s;
    STRLEN offset;
    STRLEN line;
    STRLEN column;

#ifdef UNICODE_HTML_PARSER
    #define CHR_DIST(a,b) (utf8 ? utf8_distance((U8*)(a),(U8*)(b)) : (a) - (b))
#else
    #define CHR_DIST(a,b) ((a) - (b))
#endif

    /* some events might still fire after a handler has signaled eof
     * so suppress them here.
     */
    if (p_state->eof)
	return;

    /* capture offsets */
    offset = p_state->offset;
    line = p_state->line;
    column = p_state->column;

#if 0
    {  /* used for debugging at some point */
	char *s = beg;
	int i;

	/* print debug output */
	switch(event) {
	case E_DECLARATION: printf("DECLARATION"); break;
	case E_COMMENT:     printf("COMMENT"); break;
	case E_START:       printf("START"); break;
	case E_END:         printf("END"); break;
	case E_TEXT:        printf("TEXT"); break;
	case E_PROCESS:     printf("PROCESS"); break;
	case E_NONE:        printf("NONE"); break;
	default:            printf("EVENT #%d", event); break;
	}

	printf(" [");
	while (s < end) {
	    if (*s == '\n') {
		putchar('\\'); putchar('n');
	    }
	    else
		putchar(*s);
	    s++;
	}
	printf("] %d\n", end - beg);
	for (i = 0; i < num_tokens; i++) {
	    printf("  token %d: %d %d\n",
		   i,
		   tokens[i].beg - beg,
		   tokens[i].end - tokens[i].beg);
	}
    }
#endif

    if (p_state->pending_end_tag && event != E_TEXT && event != E_COMMENT) {
	token_pos_t t;
	char dummy;
	t.beg = p_state->pending_end_tag;
	t.end = p_state->pending_end_tag + strlen(p_state->pending_end_tag);
	p_state->pending_end_tag = 0;
	report_event(p_state, E_END, &dummy, &dummy, 0, &t, 1, self);
	SPAGAIN;
    }

    /* update offsets */
    p_state->offset += CHR_DIST(end, beg);
    if (line) {
	char *s = beg;
	char *nl = NULL;
	while (s < end) {
	    if (*s == '\n') {
		p_state->line++;
		nl = s;
	    }
	    s++;
	}
	if (nl)
	    p_state->column = CHR_DIST(end, nl) - 1;
	else
	    p_state->column += CHR_DIST(end, beg);
    }

    if (event == E_NONE)
	goto IGNORE_EVENT;

#ifdef MARKED_SECTION
    if (p_state->ms == MS_IGNORE)
	goto IGNORE_EVENT;
#endif

    /* tag filters */
    if (p_state->ignore_tags || p_state->report_tags || p_state->ignore_elements) {

	if (event == E_START || event == E_END) {
	    SV* tagname = p_state->tmp;

	    assert(num_tokens >= 1);
	    sv_setpvn(tagname, tokens[0].beg, tokens[0].end - tokens[0].beg);
	    if (utf8)
		SvUTF8_on(tagname);
	    else
		SvUTF8_off(tagname);
	    if (!CASE_SENSITIVE(p_state))
		sv_lower(aTHX_ tagname);

	    if (p_state->ignoring_element) {
		if (sv_eq(p_state->ignoring_element, tagname)) {
		    if (event == E_START)
			p_state->ignore_depth++;
		    else if (--p_state->ignore_depth == 0) {
			SvREFCNT_dec(p_state->ignoring_element);
			p_state->ignoring_element = 0;
		    }
		}
		goto IGNORE_EVENT;
	    }

	    if (p_state->ignore_elements &&
		hv_fetch_ent(p_state->ignore_elements, tagname, 0, 0))
	    {
		if (event == E_START) {
		    p_state->ignoring_element = newSVsv(tagname);
		    p_state->ignore_depth = 1;
		}
		goto IGNORE_EVENT;
	    }

	    if (p_state->ignore_tags &&
		hv_fetch_ent(p_state->ignore_tags, tagname, 0, 0))
	    {
		goto IGNORE_EVENT;
	    }
	    if (p_state->report_tags &&
		!hv_fetch_ent(p_state->report_tags, tagname, 0, 0))
	    {
		goto IGNORE_EVENT;
	    }
	}
	else if (p_state->ignoring_element) {
	    goto IGNORE_EVENT;
	}
    }

    h = &p_state->handlers[event];
    if (!h->cb) {
	/* event = E_DEFAULT; */
	h = &p_state->handlers[E_DEFAULT];
	if (!h->cb)
	    goto IGNORE_EVENT;
    }

    if (SvTYPE(h->cb) != SVt_PVAV && !SvTRUE(h->cb)) {
	/* FALSE scalar ('' or 0) means IGNORE this event */
	return;
    }

    if (p_state->unbroken_text && event == E_TEXT) {
	/* should buffer text */
	if (!p_state->pend_text)
	    p_state->pend_text = newSV(256);
	if (SvOK(p_state->pend_text)) {
	    if (p_state->is_cdata != p_state->pend_text_is_cdata) {
		flush_pending_text(p_state, self);
		SPAGAIN;
		goto INIT_PEND_TEXT;
	    }
	}
	else {
	INIT_PEND_TEXT:
	    p_state->pend_text_offset = offset;
	    p_state->pend_text_line = line;
	    p_state->pend_text_column = column;
	    p_state->pend_text_is_cdata = p_state->is_cdata;
	    sv_setpvn(p_state->pend_text, "", 0);
	    if (!utf8)
		SvUTF8_off(p_state->pend_text);
	}
#ifdef UNICODE_HTML_PARSER
	if (utf8 && !SvUTF8(p_state->pend_text))
	    sv_utf8_upgrade(p_state->pend_text);
	if (utf8 || !SvUTF8(p_state->pend_text)) {
	    sv_catpvn(p_state->pend_text, beg, end - beg);
	}
	else {
	    SV *tmp = newSVpvn(beg, end - beg);
	    sv_utf8_upgrade(tmp);
	    sv_catsv(p_state->pend_text, tmp);
	    SvREFCNT_dec(tmp);
	}
#else
	sv_catpvn(p_state->pend_text, beg, end - beg);
#endif
	return;
    }
    else if (p_state->pend_text && SvOK(p_state->pend_text)) {
	flush_pending_text(p_state, self);
	SPAGAIN;
    }

    /* At this point we have decided to generate an event callback */

    argspec = h->argspec ? SvPV(h->argspec, my_na) : "";

    if (SvTYPE(h->cb) == SVt_PVAV) {

	if (*argspec == ARG_FLAG_FLAT_ARRAY) {
	    argspec++;
	    array = (AV*)h->cb;
	}
	else {
	    /* start sub-array for accumulator array */
	    array = newAV();
	}
    }
    else {
	array = 0;
	if (*argspec == ARG_FLAG_FLAT_ARRAY)
	    argspec++;

	/* start argument stack for callback */
	ENTER;
	SAVETMPS;
	PUSHMARK(SP);
    }

    for (s = argspec; *s; s++) {
	SV* arg = 0;
	int push_arg = 1;
	enum argcode argcode = (enum argcode)*s;

	switch( argcode ) {

	case ARG_SELF:
	    arg = sv_mortalcopy(self);
	    break;

	case ARG_TOKENS:
	    if (num_tokens >= 1) {
		AV* av = newAV();
		SV* prev_token = &PL_sv_undef;
		int i;
		av_extend(av, num_tokens);
		for (i = 0; i < num_tokens; i++) {
		    if (tokens[i].beg) {
			prev_token = newSVpvn(tokens[i].beg, tokens[i].end-tokens[i].beg);
			if (utf8)
			    SvUTF8_on(prev_token);
			av_push(av, prev_token);
		    }
		    else { /* boolean */
			av_push(av, p_state->bool_attr_val
				? newSVsv(p_state->bool_attr_val)
				: newSVsv(prev_token));
		    }
		}
		arg = sv_2mortal(newRV_noinc((SV*)av));
	    }
	    break;

	case ARG_TOKENPOS:
	    if (num_tokens >= 1 && tokens[0].beg >= beg) {
		AV* av = newAV();
		int i;
		av_extend(av, num_tokens*2);
		for (i = 0; i < num_tokens; i++) {
		    if (tokens[i].beg) {
			av_push(av, newSViv(CHR_DIST(tokens[i].beg, beg)));
			av_push(av, newSViv(CHR_DIST(tokens[i].end, tokens[i].beg)));
		    }
		    else { /* boolean tag value */
			av_push(av, newSViv(0));
			av_push(av, newSViv(0));
		    }
		}
		arg = sv_2mortal(newRV_noinc((SV*)av));
	    }
	    break;

	case ARG_TOKEN0:
	case ARG_TAGNAME:
	    /* fall through */

	case ARG_TAG:
	    if (num_tokens >= 1) {
		arg = sv_2mortal(newSVpvn(tokens[0].beg,
					  tokens[0].end - tokens[0].beg));
		if (utf8)
		    SvUTF8_on(arg);
		if (!CASE_SENSITIVE(p_state) && argcode != ARG_TOKEN0)
		    sv_lower(aTHX_ arg);
		if (argcode == ARG_TAG && event != E_START) {
		    char *e_type = "!##/#?#";
		    sv_insert(arg, 0, 0, &e_type[event], 1);
		}
	    }
	    break;

	case ARG_ATTR:
	case ARG_ATTRARR:
	    if (event == E_START) {
		HV* hv;
		int i;
		if (argcode == ARG_ATTR) {
		    hv = newHV();
		    arg = sv_2mortal(newRV_noinc((SV*)hv));
		}
		else {
#ifdef __GNUC__
		    /* gcc -Wall reports this variable as possibly used uninitialized */
		    hv = 0;
#endif
		    push_arg = 0;  /* deal with argument pushing here */
		}

		for (i = 1; i < num_tokens; i += 2) {
		    SV* attrname = newSVpvn(tokens[i].beg,
					    tokens[i].end-tokens[i].beg);
		    SV* attrval;

		    if (utf8)
			SvUTF8_on(attrname);
		    if (tokens[i+1].beg) {
			char *beg = tokens[i+1].beg;
			STRLEN len = tokens[i+1].end - beg;
			if (*beg == '"' || *beg == '\'' || (*beg == '`' && p_state->backquote)) {
			    assert(len >= 2 && *beg == beg[len-1]);
			    beg++; len -= 2;
			}
			attrval = newSVpvn(beg, len);
			if (utf8)
			    SvUTF8_on(attrval);
			if (!p_state->attr_encoded) {
#ifdef UNICODE_HTML_PARSER
			    if (p_state->utf8_mode) {
				sv_utf8_decode(attrval);
                                sv_utf8_upgrade(attrval);
                            }
#endif
			    decode_entities(aTHX_ attrval, p_state->entity2char, 0);
			    if (p_state->utf8_mode)
				SvUTF8_off(attrval);
			}
		    }
		    else { /* boolean */
			if (p_state->bool_attr_val)
			    attrval = newSVsv(p_state->bool_attr_val);
			else
			    attrval = newSVsv(attrname);
		    }

		    if (!CASE_SENSITIVE(p_state))
			sv_lower(aTHX_ attrname);

		    if (argcode == ARG_ATTR) {
			if (hv_exists_ent(hv, attrname, 0) ||
			    !hv_store_ent(hv, attrname, attrval, 0)) {
			    SvREFCNT_dec(attrval);
			}
			SvREFCNT_dec(attrname);
		    }
		    else { /* ARG_ATTRARR */
			if (array) {
			    av_push(array, attrname);
			    av_push(array, attrval);
			}
			else {
			    XPUSHs(sv_2mortal(attrname));
			    XPUSHs(sv_2mortal(attrval));
			}
		    }
		}
	    }
	    else if (argcode == ARG_ATTRARR) {
		push_arg = 0;
	    }
	    break;

	case ARG_ATTRSEQ:       /* (v2 compatibility stuff) */
	    if (event == E_START) {
		AV* av = newAV();
		int i;
		for (i = 1; i < num_tokens; i += 2) {
		    SV* attrname = newSVpvn(tokens[i].beg,
					    tokens[i].end-tokens[i].beg);
		    if (utf8)
			SvUTF8_on(attrname);
		    if (!CASE_SENSITIVE(p_state))
			sv_lower(aTHX_ attrname);
		    av_push(av, attrname);
		}
		arg = sv_2mortal(newRV_noinc((SV*)av));
	    }
	    break;

	case ARG_TEXT:
	    arg = sv_2mortal(newSVpvn(beg, end - beg));
	    if (utf8)
		SvUTF8_on(arg);
	    break;

	case ARG_DTEXT:
	    if (event == E_TEXT) {
		arg = sv_2mortal(newSVpvn(beg, end - beg));
		if (utf8)
		    SvUTF8_on(arg);
		if (!p_state->is_cdata) {
#ifdef UNICODE_HTML_PARSER
		    if (p_state->utf8_mode) {
			sv_utf8_decode(arg);
                        sv_utf8_upgrade(arg);
                    }
#endif
		    decode_entities(aTHX_ arg, p_state->entity2char, 1);
		    if (p_state->utf8_mode)
			SvUTF8_off(arg);
		}
	    }
	    break;

	case ARG_IS_CDATA:
	    if (event == E_TEXT) {
		arg = boolSV(p_state->is_cdata);
	    }
	    break;

        case ARG_SKIPPED_TEXT:
	    arg = sv_2mortal(p_state->skipped_text);
	    p_state->skipped_text = newSVpvn("", 0);
            break;

	case ARG_OFFSET:
	    arg = sv_2mortal(newSViv(offset));
	    break;

	case ARG_OFFSET_END:
	    arg = sv_2mortal(newSViv(offset + CHR_DIST(end, beg)));
	    break;

	case ARG_LENGTH:
	    arg = sv_2mortal(newSViv(CHR_DIST(end, beg)));
	    break;

	case ARG_LINE:
	    arg = sv_2mortal(newSViv(line));
	    break;

	case ARG_COLUMN:
	    arg = sv_2mortal(newSViv(column));
	    break;

	case ARG_EVENT:
	    assert(event >= 0 && event < EVENT_COUNT);
	    arg = sv_2mortal(newSVpv(event_id_str[event], 0));
	    break;

	case ARG_LITERAL:
	{
	    int len = (unsigned char)s[1];
	    arg = sv_2mortal(newSVpvn(s+2, len));
	    if (SvUTF8(h->argspec))
		SvUTF8_on(arg);
	    s += len + 1;
	}
	break;

	case ARG_UNDEF:
	    arg = sv_mortalcopy(&PL_sv_undef);
	    break;

	default:
	    arg = sv_2mortal(newSVpvf("Bad argspec %d", *s));
	    break;
	}

	if (push_arg) {
	    if (!arg)
		arg = sv_mortalcopy(&PL_sv_undef);

	    if (array) {
		/* have to fix mortality here or add mortality to
		 * XPUSHs after removing it from the switch cases.
		 */
		av_push(array, SvREFCNT_inc(arg));
	    }
	    else {
		XPUSHs(arg);
	    }
	}
    }

    if (array) {
	if (array != (AV*)h->cb)
	    av_push((AV*)h->cb, newRV_noinc((SV*)array));
    }
    else {
	PUTBACK;

	if ((enum argcode)*argspec == ARG_SELF && !SvROK(h->cb)) {
	    char *method = SvPV(h->cb, my_na);
	    perl_call_method(method, G_DISCARD | G_EVAL | G_VOID);
	}
	else {
	    perl_call_sv(h->cb, G_DISCARD | G_EVAL | G_VOID);
	}

	if (SvTRUE(ERRSV)) {
	    RETHROW;
	}

	FREETMPS;
	LEAVE;
    }
    if (p_state->skipped_text)
	SvCUR_set(p_state->skipped_text, 0);
    return;

IGNORE_EVENT:
    if (p_state->skipped_text) {
	if (event != E_TEXT && p_state->pend_text && SvOK(p_state->pend_text))
	    flush_pending_text(p_state, self);
#ifdef UNICODE_HTML_PARSER
	if (utf8 && !SvUTF8(p_state->skipped_text))
	    sv_utf8_upgrade(p_state->skipped_text);
	if (utf8 || !SvUTF8(p_state->skipped_text)) {
#endif
	    sv_catpvn(p_state->skipped_text, beg, end - beg);
#ifdef UNICODE_HTML_PARSER
	}
	else {
	    SV *tmp = newSVpvn(beg, end - beg);
	    sv_utf8_upgrade(tmp);
	    sv_catsv(p_state->skipped_text, tmp);
	    SvREFCNT_dec(tmp);
	}
#endif
    }
#undef CHR_DIST
    return;
}


EXTERN SV*
argspec_compile(SV* src, PSTATE* p_state)
{
    dTHX;
    SV* argspec = newSVpvn("", 0);
    STRLEN len;
    char *s = SvPV(src, len);
    char *end = s + len;

    if (SvUTF8(src))
	SvUTF8_on(argspec);

    while (isHSPACE(*s))
	s++;

    if (*s == '@') {
	/* try to deal with '@{ ... }' wrapping */
	char *tmp = s + 1;
	while (isHSPACE(*tmp))
	    tmp++;
	if (*tmp == '{') {
	    char c = ARG_FLAG_FLAT_ARRAY;
	    sv_catpvn(argspec, &c, 1);
	    tmp++;
	    while (isHSPACE(*tmp))
		tmp++;
	    s = tmp;
	}
    }
    while (s < end) {
	if (isHNAME_FIRST(*s) || *s == '@') {
	    char *name = s;
	    int a = ARG_SELF;
	    const char * const *arg_name;

	    s++;
	    while (isHNAME_CHAR(*s))
		s++;

	    /* check identifier */
	    for ( arg_name = argname; a < ARG_LITERAL ; ++a, ++arg_name ) {
		if (strnEQ(*arg_name, name, s - name) &&
		    (*arg_name)[s - name] == '\0')
		    break;
	    }
	    if (a < ARG_LITERAL) {
		char c = (unsigned char) a;
		sv_catpvn(argspec, &c, 1);

		if (a == ARG_LINE || a == ARG_COLUMN) {
		    if (!p_state->line)
			p_state->line = 1; /* enable tracing of line/column */
		}
		if (a == ARG_SKIPPED_TEXT) {
		    if (!p_state->skipped_text) {
			p_state->skipped_text = newSVpvn("", 0);
                    }
                }
		if (a == ARG_ATTR || a == ARG_ATTRARR) {
		    if (p_state->argspec_entity_decode != ARG_DTEXT)
			p_state->argspec_entity_decode = ARG_ATTR;
		}
		else if (a == ARG_DTEXT) {
		    p_state->argspec_entity_decode = ARG_DTEXT;
		}
	    }
	    else {
		croak("Unrecognized identifier %.*s in argspec", (int) (s - name), name);
	    }
	}
	else if (*s == '"' || *s == '\'') {
	    char *string_beg = s;
	    s++;
	    while (s < end && *s != *string_beg && *s != '\\')
		s++;
	    if (*s == *string_beg) {
		/* literal */
		int len = s - string_beg - 1;
		unsigned char buf[2];
		if (len > 255)
		    croak("Literal string is longer than 255 chars in argspec");
		buf[0] = ARG_LITERAL;
		buf[1] = len;
		sv_catpvn(argspec, (char*)buf, 2);
		sv_catpvn(argspec, string_beg+1, len);
		s++;
	    }
	    else if (*s == '\\') {
		croak("Backslash reserved for literal string in argspec");
	    }
	    else {
		croak("Unterminated literal string in argspec");
	    }
	}
	else {
	    croak("Bad argspec (%s)", s);
	}

	while (isHSPACE(*s))
	    s++;

	if (*s == '}' && SvPVX(argspec)[0] == ARG_FLAG_FLAT_ARRAY) {
	    /* end of '@{ ... }' */
	    s++;
	    while (isHSPACE(*s))
		s++;
	    if (s < end)
		croak("Bad argspec: stuff after @{...} (%s)", s);
	}

	if (s == end)
	    break;
	if (*s != ',') {
	    croak("Missing comma separator in argspec");
	}
	s++;
	while (isHSPACE(*s))
	    s++;
    }
    return argspec;
}


static void
flush_pending_text(PSTATE* p_state, SV* self)
{
    dTHX;
    bool   old_unbroken_text = p_state->unbroken_text;
    SV*    old_pend_text     = p_state->pend_text;
    bool   old_is_cdata      = p_state->is_cdata;
    STRLEN old_offset        = p_state->offset;
    STRLEN old_line          = p_state->line;
    STRLEN old_column        = p_state->column;

    assert(p_state->pend_text && SvOK(p_state->pend_text));

    p_state->unbroken_text = 0;
    p_state->pend_text     = 0;
    p_state->is_cdata      = p_state->pend_text_is_cdata;
    p_state->offset        = p_state->pend_text_offset;
    p_state->line          = p_state->pend_text_line;
    p_state->column        = p_state->pend_text_column;

    report_event(p_state, E_TEXT,
		 SvPVX(old_pend_text), SvEND(old_pend_text),
		 SvUTF8(old_pend_text), 0, 0, self);
    SvOK_off(old_pend_text);

    p_state->unbroken_text = old_unbroken_text;
    p_state->pend_text     = old_pend_text;
    p_state->is_cdata      = old_is_cdata;
    p_state->offset        = old_offset;
    p_state->line          = old_line;
    p_state->column        = old_column;
}

static char*
skip_until_gt(char *beg, char *end)
{
    /* tries to emulate quote skipping behaviour observed in MSIE */
    char *s = beg;
    char quote = '\0';
    char prev = ' ';
    while (s < end) {
	if (!quote && *s == '>')
	    return s;
	if (*s == '"' || *s == '\'') {
	    if (*s == quote) {
		quote = '\0';  /* end of quoted string */
	    }
	    else if (!quote && (prev == ' ' || prev == '=')) {
		quote = *s;
	    }
	}
	prev = *s++;
    }
    return end;
}

static char*
parse_comment(PSTATE* p_state, char *beg, char *end, U32 utf8, SV* self)
{
    char *s = beg;

    if (p_state->strict_comment) {
	dTOKENS(4);
	char *start_com = s;  /* also used to signal inside/outside */

	while (1) {
	    /* try to locate "--" */
	FIND_DASH_DASH:
	    /* printf("find_dash_dash: [%s]\n", s); */
	    while (s < end && *s != '-' && *s != '>')
		s++;

	    if (s == end) {
		FREE_TOKENS;
		return beg;
	    }

	    if (*s == '>') {
		s++;
		if (start_com)
		    goto FIND_DASH_DASH;

		/* we are done recognizing all comments, make callbacks */
		report_event(p_state, E_COMMENT,
			     beg - 4, s, utf8,
			     tokens, num_tokens,
			     self);
		FREE_TOKENS;

		return s;
	    }

	    s++;
	    if (s == end) {
		FREE_TOKENS;
		return beg;
	    }

	    if (*s == '-') {
		/* two dashes in a row seen */
		s++;
		/* do something */
		if (start_com) {
		    PUSH_TOKEN(start_com, s-2);
		    start_com = 0;
		}
		else {
		    start_com = s;
		}
	    }
	}
    }
    else if (p_state->no_dash_dash_comment_end) {
	token_pos_t token;
        token.beg = beg;
        /* a lone '>' signals end-of-comment */
	while (s < end && *s != '>')
	    s++;
	token.end = s;
	if (s < end) {
	    s++;
	    report_event(p_state, E_COMMENT, beg-4, s, utf8, &token, 1, self);
	    return s;
	}
	else {
	    return beg;
	}
    }
    else { /* non-strict comment */
	token_pos_t token;
	token.beg = beg;
	/* try to locate /--\s*>/ which signals end-of-comment */
    LOCATE_END:
	while (s < end && *s != '-')
	    s++;
	token.end = s;
	if (s < end) {
	    s++;
	    if (*s == '-') {
		s++;
		while (isHSPACE(*s))
		    s++;
		if (*s == '>') {
		    s++;
		    /* yup */
		    report_event(p_state, E_COMMENT, beg-4, s, utf8, &token, 1, self);
		    return s;
		}
	    }
	    if (s < end) {
		s = token.end + 1;
		goto LOCATE_END;
	    }
	}

	if (s == end)
	    return beg;
    }

    return 0;
}


#ifdef MARKED_SECTION

static void
marked_section_update(PSTATE* p_state)
{
    dTHX;
    /* we look at p_state->ms_stack to determine p_state->ms */
    AV* ms_stack = p_state->ms_stack;
    p_state->ms = MS_NONE;

    if (ms_stack) {
	int stack_len = av_len(ms_stack);
	int stack_idx;
	for (stack_idx = 0; stack_idx <= stack_len; stack_idx++) {
	    SV** svp = av_fetch(ms_stack, stack_idx, 0);
	    if (svp) {
		AV* tokens = (AV*)SvRV(*svp);
		int tokens_len = av_len(tokens);
		int i;
		assert(SvTYPE(tokens) == SVt_PVAV);
		for (i = 0; i <= tokens_len; i++) {
		    SV** svp = av_fetch(tokens, i, 0);
		    if (svp) {
			STRLEN len;
			char *token_str = SvPV(*svp, len);
			enum marked_section_t token;
			if (strEQ(token_str, "include"))
			    token = MS_INCLUDE;
			else if (strEQ(token_str, "rcdata"))
			    token = MS_RCDATA;
			else if (strEQ(token_str, "cdata"))
			    token = MS_CDATA;
			else if (strEQ(token_str, "ignore"))
			    token = MS_IGNORE;
			else
			    token = MS_NONE;
			if (p_state->ms < token)
			    p_state->ms = token;
		    }
		}
	    }
	}
    }
    /* printf("MS %d\n", p_state->ms); */
    p_state->is_cdata = (p_state->ms == MS_CDATA);
    return;
}


static char*
parse_marked_section(PSTATE* p_state, char *beg, char *end, U32 utf8, SV* self)
{
    dTHX;
    char *s;
    AV* tokens = 0;

    if (!p_state->marked_sections)
	return 0;

    assert(beg[0] == '<');
    assert(beg[1] == '!');
    assert(beg[2] == '[');
    s = beg + 3;

FIND_NAMES:
    while (isHSPACE(*s))
	s++;
    while (isHNAME_FIRST(*s)) {
	char *name_start = s;
	char *name_end;
	SV *name;
	s++;
	while (isHNAME_CHAR(*s))
	    s++;
	name_end = s;
	while (isHSPACE(*s))
	    s++;
	if (s == end)
	    goto PREMATURE;

	if (!tokens)
	    tokens = newAV();
	name = newSVpvn(name_start, name_end - name_start);
	if (utf8)
	    SvUTF8_on(name);
	av_push(tokens, sv_lower(aTHX_ name));
    }
    if (*s == '-') {
	s++;
	if (*s == '-') {
	    /* comment */
	    s++;
	    while (1) {
		while (s < end && *s != '-')
		    s++;
		if (s == end)
		    goto PREMATURE;

		s++;  /* skip first '-' */
		if (*s == '-') {
		    s++;
		    /* comment finished */
		    goto FIND_NAMES;
		}
	    }
	}
	else
	    goto FAIL;

    }
    if (*s == '[') {
	s++;
	/* yup */

	if (!tokens) {
	    tokens = newAV();
	    av_push(tokens, newSVpvn("include", 7));
	}

	if (!p_state->ms_stack)
	    p_state->ms_stack = newAV();
	av_push(p_state->ms_stack, newRV_noinc((SV*)tokens));
	marked_section_update(p_state);
	report_event(p_state, E_NONE, beg, s, utf8, 0, 0, self);
	return s;
    }

FAIL:
    SvREFCNT_dec(tokens);
    return 0; /* not yet implemented */

PREMATURE:
    SvREFCNT_dec(tokens);
    return beg;
}
#endif


static char*
parse_decl(PSTATE* p_state, char *beg, char *end, U32 utf8, SV* self)
{
    char *s = beg + 2;

    if (*s == '-') {
	/* comment? */

	char *tmp;
	s++;
	if (s == end)
	    return beg;

	if (*s != '-')
	    goto DECL_FAIL;  /* nope, illegal */

	/* yes, two dashes seen */
	s++;

	tmp = parse_comment(p_state, s, end, utf8, self);
	return (tmp == s) ? beg : tmp;
    }

#ifdef MARKED_SECTION
    if (*s == '[') {
	/* marked section */
	char *tmp;
	tmp = parse_marked_section(p_state, beg, end, utf8, self);
	if (!tmp)
	    goto DECL_FAIL;
	return tmp;
    }
#endif

    if (*s == '>') {
	/* make <!> into empty comment <SGML Handbook 36:32> */
	token_pos_t token;
	token.beg = s;
	token.end = s;
	s++;
	report_event(p_state, E_COMMENT, beg, s, utf8, &token, 1, self);
	return s;
    }

    if (isALPHA(*s)) {
	dTOKENS(8);
	char *decl_id = s;
	STRLEN decl_id_len;

	s++;
	/* declaration */
	while (s < end && isHNAME_CHAR(*s))
	    s++;
	decl_id_len = s - decl_id;
	if (s == end)
	    goto PREMATURE;

	/* just hardcode a few names as the recognized declarations */
	if (!((decl_id_len == 7 &&
	       strnEQx(decl_id, "DOCTYPE", 7, !CASE_SENSITIVE(p_state))) ||
	      (decl_id_len == 6 &&
	       strnEQx(decl_id, "ENTITY",  6, !CASE_SENSITIVE(p_state)))
	    )
	    )
	{
	    goto FAIL;
	}

	/* first word available */
	PUSH_TOKEN(decl_id, s);

	while (1) {
	    while (s < end && isHSPACE(*s))
		s++;

	    if (s == end)
		goto PREMATURE;

	    if (*s == '"' || *s == '\'' || (*s == '`' && p_state->backquote)) {
		char *str_beg = s;
		s++;
		while (s < end && *s != *str_beg)
		    s++;
		if (s == end)
		    goto PREMATURE;
		s++;
		PUSH_TOKEN(str_beg, s);
	    }
	    else if (*s == '-') {
		/* comment */
		char *com_beg = s;
		s++;
		if (s == end)
		    goto PREMATURE;
		if (*s != '-')
		    goto FAIL;
		s++;

		while (1) {
		    while (s < end && *s != '-')
			s++;
		    if (s == end)
			goto PREMATURE;
		    s++;
		    if (s == end)
			goto PREMATURE;
		    if (*s == '-') {
			s++;
			PUSH_TOKEN(com_beg, s);
			break;
		    }
		}
	    }
	    else if (*s != '>') {
		/* plain word */
		char *word_beg = s;
		s++;
		while (s < end && isHNOT_SPACE_GT(*s))
		    s++;
		if (s == end)
		    goto PREMATURE;
		PUSH_TOKEN(word_beg, s);
	    }
	    else {
		break;
	    }
	}

	if (s == end)
	    goto PREMATURE;
	if (*s == '>') {
	    s++;
	    report_event(p_state, E_DECLARATION, beg, s, utf8, tokens, num_tokens, self);
	    FREE_TOKENS;
	    return s;
	}

    FAIL:
	FREE_TOKENS;
	goto DECL_FAIL;

    PREMATURE:
	FREE_TOKENS;
	return beg;

    }

DECL_FAIL:
    if (p_state->strict_comment)
	return 0;

    /* consider everything up to the first '>' a comment */
    while (s < end && *s != '>')
	s++;
    if (s < end) {
	token_pos_t token;
	token.beg = beg + 2;
	token.end = s;
	s++;
	report_event(p_state, E_COMMENT, beg, s, utf8, &token, 1, self);
	return s;
    }
    else {
	return beg;
    }
}


static char*
parse_start(PSTATE* p_state, char *beg, char *end, U32 utf8, SV* self)
{
    char *s = beg;
    int empty_tag = 0;
    dTOKENS(16);

    hctype_t tag_name_first, tag_name_char;
    hctype_t attr_name_first, attr_name_char;

    if (STRICT_NAMES(p_state)) {
	tag_name_first = attr_name_first = HCTYPE_NAME_FIRST;
	tag_name_char  = attr_name_char  = HCTYPE_NAME_CHAR;
    }
    else {
	tag_name_first = tag_name_char = HCTYPE_NOT_SPACE_GT;
	attr_name_first = HCTYPE_NOT_SPACE_GT;
	attr_name_char  = HCTYPE_NOT_SPACE_EQ_GT;
    }

    s += 2;

    while (s < end && isHCTYPE(*s, tag_name_char)) {
	if (*s == '/' && ALLOW_EMPTY_TAG(p_state)) {
	    if ((s + 1) == end)
		goto PREMATURE;
	    if (*(s + 1) == '>')
		break;
	}
	s++;
    }
    PUSH_TOKEN(beg+1, s);  /* tagname */

    while (isHSPACE(*s))
	s++;
    if (s == end)
	goto PREMATURE;

    while (isHCTYPE(*s, attr_name_first)) {
	/* attribute */
	char *attr_name_beg = s;
	char *attr_name_end;
	if (*s == '/' && ALLOW_EMPTY_TAG(p_state)) {
	    if ((s + 1) == end)
		goto PREMATURE;
	    if (*(s + 1) == '>')
		break;
	}
	s++;
	while (s < end && isHCTYPE(*s, attr_name_char)) {
	    if (*s == '/' && ALLOW_EMPTY_TAG(p_state)) {
		if ((s + 1) == end)
		    goto PREMATURE;
		if (*(s + 1) == '>')
		    break;
	    }
	    s++;
	}
	if (s == end)
	    goto PREMATURE;

	attr_name_end = s;
	PUSH_TOKEN(attr_name_beg, attr_name_end); /* attr name */

	while (isHSPACE(*s))
	    s++;
	if (s == end)
	    goto PREMATURE;

	if (*s == '=') {
	    /* with a value */
	    s++;
	    while (isHSPACE(*s))
		s++;
	    if (s == end)
		goto PREMATURE;
	    if (*s == '>') {
		/* parse it similar to ="" */
		PUSH_TOKEN(s, s);
		break;
	    }
	    if (*s == '"' || *s == '\'' || (*s == '`' && p_state->backquote)) {
		char *str_beg = s;
		s++;
		while (s < end && *s != *str_beg)
		    s++;
		if (s == end)
		    goto PREMATURE;
		s++;
		PUSH_TOKEN(str_beg, s);
	    }
	    else {
		char *word_start = s;
		while (s < end && isHNOT_SPACE_GT(*s)) {
		    if (*s == '/' && ALLOW_EMPTY_TAG(p_state)) {
			if ((s + 1) == end)
			    goto PREMATURE;
			if (*(s + 1) == '>')
			    break;
		    }
		    s++;
		}
		if (s == end)
		    goto PREMATURE;
		PUSH_TOKEN(word_start, s);
	    }
	    while (isHSPACE(*s))
		s++;
	    if (s == end)
		goto PREMATURE;
	}
	else {
	    PUSH_TOKEN(0, 0); /* boolean attr value */
	}
    }

    if (ALLOW_EMPTY_TAG(p_state) && *s == '/') {
	s++;
	if (s == end)
	    goto PREMATURE;
	empty_tag = 1;
    }

    if (*s == '>') {
	s++;
	/* done */
	report_event(p_state, E_START, beg, s, utf8, tokens, num_tokens, self);
	if (empty_tag) {
	    report_event(p_state, E_END, s, s, utf8, tokens, 1, self);
	}
	else if (!p_state->xml_mode) {
	    /* find out if this start tag should put us into literal_mode
	     */
	    int i;
	    int tag_len = tokens[0].end - tokens[0].beg;

	    for (i = 0; literal_mode_elem[i].len; i++) {
		if (tag_len == literal_mode_elem[i].len) {
		    /* try to match it */
		    char *s = beg + 1;
		    char *t = literal_mode_elem[i].str;
		    int len = tag_len;
		    while (len) {
			if (toLOWER(*s) != *t)
			    break;
			s++;
			t++;
			if (!--len) {
			    /* found it */
			    p_state->literal_mode = literal_mode_elem[i].str;
			    p_state->is_cdata = literal_mode_elem[i].is_cdata;
			    /* printf("Found %s\n", p_state->literal_mode); */
			    goto END_OF_LITERAL_SEARCH;
			}
		    }
		}
	    }
	END_OF_LITERAL_SEARCH:
	    ;
	}

	FREE_TOKENS;
	return s;
    }

    FREE_TOKENS;
    return 0;

PREMATURE:
    FREE_TOKENS;
    return beg;
}


static char*
parse_end(PSTATE* p_state, char *beg, char *end, U32 utf8, SV* self)
{
    char *s = beg+2;
    hctype_t name_first, name_char;

    if (STRICT_NAMES(p_state)) {
	name_first = HCTYPE_NAME_FIRST;
	name_char  = HCTYPE_NAME_CHAR;
    }
    else {
	name_first = name_char = HCTYPE_NOT_SPACE_GT;
    }

    if (isHCTYPE(*s, name_first)) {
	token_pos_t tagname;
	tagname.beg = s;
	s++;
	while (s < end && isHCTYPE(*s, name_char))
	    s++;
	tagname.end = s;

	if (p_state->strict_end) {
	    while (isHSPACE(*s))
		s++;
	}
	else {
	    s = skip_until_gt(s, end);
	}
	if (s < end) {
	    if (*s == '>') {
		s++;
		/* a complete end tag has been recognized */
		report_event(p_state, E_END, beg, s, utf8, &tagname, 1, self);
		return s;
	    }
	}
	else {
	    return beg;
	}
    }
    else if (!p_state->strict_comment) {
	s = skip_until_gt(s, end);
	if (s < end) {
	    token_pos_t token;
	    token.beg = beg + 2;
	    token.end = s;
	    s++;
	    report_event(p_state, E_COMMENT, beg, s, utf8, &token, 1, self);
	    return s;
	}
	else {
	    return beg;
	}
    }
    return 0;
}


static char*
parse_process(PSTATE* p_state, char *beg, char *end, U32 utf8, SV* self)
{
    char *s = beg + 2;  /* skip '<?' */
    /* processing instruction */
    token_pos_t token_pos;
    token_pos.beg = s;

    while (s < end) {
	if (*s == '>') {
	    token_pos.end = s;
	    s++;

	    if (p_state->xml_mode || p_state->xml_pic) {
		/* XML processing instructions are ended by "?>" */
		if (s - beg < 4 || s[-2] != '?')
		    continue;
		token_pos.end = s - 2;
	    }

	    /* a complete processing instruction seen */
	    report_event(p_state, E_PROCESS, beg, s, utf8,
			 &token_pos, 1, self);
	    return s;
	}
	s++;
    }
    return beg;  /* could not find end */
}


#ifdef USE_PFUNC
static char*
parse_null(PSTATE* p_state, char *beg, char *end, U32 utf8, SV* self)
{
    return 0;
}



#include "pfunc.h"                   /* declares the parsefunc[] */
#endif /* USE_PFUNC */

static char*
parse_buf(pTHX_ PSTATE* p_state, char *beg, char *end, U32 utf8, SV* self)
{
    char *s = beg;
    char *t = beg;
    char *new_pos;

    while (!p_state->eof) {
	/*
	 * At the start of this loop we will always be ready for eating text
	 * or a new tag.  We will never be inside some tag.  The 't' points
	 * to where we started and the 's' is advanced as we go.
	 */

	while (p_state->literal_mode) {
	    char *l = p_state->literal_mode;
	    char *end_text;

	    while (s < end && *s != '<') {
		s++;
	    }

	    if (s == end) {
		s = t;
		goto DONE;
	    }

	    end_text = s;
	    s++;

	    /* here we rely on '\0' termination of perl svpv buffers */
	    if (*s == '/') {
		s++;
		while (*l && toLOWER(*s) == *l) {
		    s++;
		    l++;
		}

		if (!*l && (strNE(p_state->literal_mode, "plaintext") || p_state->closing_plaintext)) {
		    /* matched it all */
		    token_pos_t end_token;
		    end_token.beg = end_text + 2;
		    end_token.end = s;

		    while (isHSPACE(*s))
			s++;
		    if (*s == '>') {
			s++;
			if (t != end_text)
			    report_event(p_state, E_TEXT, t, end_text, utf8,
					 0, 0, self);
			report_event(p_state, E_END,  end_text, s, utf8,
				     &end_token, 1, self);
			p_state->literal_mode = 0;
			p_state->is_cdata = 0;
			t = s;
		    }
		}
	    }
	}

#ifdef MARKED_SECTION
	while (p_state->ms == MS_CDATA || p_state->ms == MS_RCDATA) {
	    while (s < end && *s != ']')
		s++;
	    if (*s == ']') {
		char *end_text = s;
		s++;
		if (*s == ']' && *(s + 1) == '>') {
		    s += 2;
		    /* marked section end */
		    if (t != end_text)
			report_event(p_state, E_TEXT, t, end_text, utf8,
				     0, 0, self);
		    report_event(p_state, E_NONE, end_text, s, utf8, 0, 0, self);
		    t = s;
		    SvREFCNT_dec(av_pop(p_state->ms_stack));
		    marked_section_update(p_state);
		    continue;
		}
	    }
	    if (s == end) {
		s = t;
		goto DONE;
	    }
	}
#endif

	/* first we try to match as much text as possible */
	while (s < end && *s != '<') {
#ifdef MARKED_SECTION
	    if (p_state->ms && *s == ']') {
		char *end_text = s;
		s++;
		if (*s == ']') {
		    s++;
		    if (*s == '>') {
			s++;
			report_event(p_state, E_TEXT, t, end_text, utf8,
				     0, 0, self);
			report_event(p_state, E_NONE, end_text, s, utf8,
				     0, 0, self);
			t = s;
			SvREFCNT_dec(av_pop(p_state->ms_stack));
			marked_section_update(p_state);
			continue;
		    }
		}
	    }
#endif
	    s++;
	}
	if (s != t) {
	    if (*s == '<') {
		report_event(p_state, E_TEXT, t, s, utf8, 0, 0, self);
		t = s;
	    }
	    else {
		s--;
		if (isHSPACE(*s)) {
		    /* wait with white space at end */
		    while (s >= t && isHSPACE(*s))
			s--;
		}
		else {
		    /* might be a chopped up entities/words */
		    while (s >= t && !isHSPACE(*s))
			s--;
		    while (s >= t && isHSPACE(*s))
			s--;
		}
		s++;
		if (s != t)
		    report_event(p_state, E_TEXT, t, s, utf8, 0, 0, self);
		break;
	    }
	}

	if (end - s < 3)
	    break;

	/* next char is known to be '<' and pointed to by 't' as well as 's' */
	s++;

#ifdef USE_PFUNC
	new_pos = parsefunc[(unsigned char)*s](p_state, t, end, utf8, self);
#else
	if (isHNAME_FIRST(*s))
	    new_pos = parse_start(p_state, t, end, utf8, self);
	else if (*s == '/')
	    new_pos = parse_end(p_state, t, end, utf8, self);
	else if (*s == '!')
	    new_pos = parse_decl(p_state, t, end, utf8, self);
	else if (*s == '?')
	    new_pos = parse_process(p_state, t, end, utf8, self);
	else
	    new_pos = 0;
#endif /* USE_PFUNC */

	if (new_pos) {
	    if (new_pos == t) {
		/* no progress, need more data to know what it is */
		s = t;
		break;
	    }
	    t = s = new_pos;
	}

	/* if we get out here then this was not a conforming tag, so
	 * treat it is plain text at the top of the loop again (we
	 * have already skipped past the "<").
	 */
    }

DONE:
    return s;

}

EXTERN void
parse(pTHX_
      PSTATE* p_state,
      SV* chunk,
      SV* self)
{
    char *s, *beg, *end;
    U32 utf8 = 0;
    STRLEN len;

    if (!p_state->start_document) {
	char dummy[1];
	report_event(p_state, E_START_DOCUMENT, dummy, dummy, 0, 0, 0, self);
	p_state->start_document = 1;
    }

    if (!chunk) {
	/* eof */
	char empty[1];
	if (p_state->buf && SvOK(p_state->buf)) {
	    /* flush it */
	    s = SvPV(p_state->buf, len);
	    end = s + len;
	    utf8 = SvUTF8(p_state->buf);
	    assert(len);

	    while (s < end) {
		if (p_state->literal_mode) {
		    if (strEQ(p_state->literal_mode, "plaintext") ||
			strEQ(p_state->literal_mode, "xmp") ||
			strEQ(p_state->literal_mode, "iframe") ||
			strEQ(p_state->literal_mode, "textarea"))
		    {
			/* rest is considered text */
			break;
                    }
		    if (strEQ(p_state->literal_mode, "script") ||
			strEQ(p_state->literal_mode, "style"))
		    {
			/* effectively make it an empty element */
			token_pos_t t;
			char dummy;
			t.beg = p_state->literal_mode;
			t.end = p_state->literal_mode + strlen(p_state->literal_mode);
			report_event(p_state, E_END, &dummy, &dummy, 0, &t, 1, self);
		    }
		    else {
			p_state->pending_end_tag = p_state->literal_mode;
		    }
		    p_state->literal_mode = 0;
		    s = parse_buf(aTHX_ p_state, s, end, utf8, self);
		    continue;
		}

		if (!p_state->strict_comment && !p_state->no_dash_dash_comment_end && *s == '<') {
		    p_state->no_dash_dash_comment_end = 1;
		    s = parse_buf(aTHX_ p_state, s, end, utf8, self);
		    continue;
		}

		if (!p_state->strict_comment && *s == '<') {
		    char *s1 = s + 1;
		    if (s1 == end || isHNAME_FIRST(*s1) || *s1 == '/' || *s1 == '!' || *s1 == '?') {
			/* some kind of unterminated markup.  Report rest as as comment */
			token_pos_t token;
			token.beg = s + 1;
			token.end = end;
			report_event(p_state, E_COMMENT, s, end, utf8, &token, 1, self);
			s = end;
		    }
		}

		break;
	    }

	    if (s < end) {
		/* report rest as text */
		report_event(p_state, E_TEXT, s, end, utf8, 0, 0, self);
	    }

	    SvREFCNT_dec(p_state->buf);
	    p_state->buf = 0;
	}
	if (p_state->pend_text && SvOK(p_state->pend_text))
	    flush_pending_text(p_state, self);

	if (p_state->ignoring_element) {
	    /* document not balanced */
	    SvREFCNT_dec(p_state->ignoring_element);
	    p_state->ignoring_element = 0;
	}
	report_event(p_state, E_END_DOCUMENT, empty, empty, 0, 0, 0, self);

	/* reset state */
	p_state->offset = 0;
	if (p_state->line)
	    p_state->line = 1;
	p_state->column = 0;
	p_state->start_document = 0;
	p_state->literal_mode = 0;
	p_state->is_cdata = 0;
	return;
    }

#ifdef UNICODE_HTML_PARSER
    if (p_state->utf8_mode)
	sv_utf8_downgrade(chunk, 0);
#endif

    if (p_state->buf && SvOK(p_state->buf)) {
	sv_catsv(p_state->buf, chunk);
	beg = SvPV(p_state->buf, len);
	utf8 = SvUTF8(p_state->buf);
    }
    else {
	beg = SvPV(chunk, len);
	utf8 = SvUTF8(chunk);
	if (p_state->offset == 0 && DOWARN) {
	    /* Print warnings if we find unexpected Unicode BOM forms */
#ifdef UNICODE_HTML_PARSER
	    if (p_state->argspec_entity_decode &&
		!(p_state->attr_encoded && p_state->argspec_entity_decode == ARG_ATTR) &&
		!p_state->utf8_mode && (
                 (!utf8 && len >= 3 && strnEQ(beg, "\xEF\xBB\xBF", 3)) ||
		 (utf8 && len >= 6 && strnEQ(beg, "\xC3\xAF\xC2\xBB\xC2\xBF", 6)) ||
		 (!utf8 && probably_utf8_chunk(aTHX_ beg, len))
		)
	       )
	    {
		warn("Parsing of undecoded UTF-8 will give garbage when decoding entities");
	    }
	    if (utf8 && len >= 2 && strnEQ(beg, "\xFF\xFE", 2)) {
		warn("Parsing string decoded with wrong endianness");
	    }
#endif
	    if (!utf8 && len >= 4 &&
		(strnEQ(beg, "\x00\x00\xFE\xFF", 4) ||
		 strnEQ(beg, "\xFE\xFF\x00\x00", 4))
		)
	    {
		warn("Parsing of undecoded UTF-32");
	    }
	    else if (!utf8 && len >= 2 &&
		     (strnEQ(beg, "\xFE\xFF", 2) || strnEQ(beg, "\xFF\xFE", 2))
		)
	    {
		warn("Parsing of undecoded UTF-16");
	    }
	}
    }

    if (!len)
	return; /* nothing to do */

    end = beg + len;
    s = parse_buf(aTHX_ p_state, beg, end, utf8, self);

    if (s == end || p_state->eof) {
	if (p_state->buf) {
	    SvOK_off(p_state->buf);
	}
    }
    else {
	/* need to keep rest in buffer */
	if (p_state->buf) {
	    /* chop off some chars at the beginning */
	    if (SvOK(p_state->buf)) {
		sv_chop(p_state->buf, s);
	    }
	    else {
		sv_setpvn(p_state->buf, s, end - s);
		if (utf8)
		    SvUTF8_on(p_state->buf);
		else
		    SvUTF8_off(p_state->buf);
	    }
	}
	else {
	    p_state->buf = newSVpv(s, end - s);
	    if (utf8)
		SvUTF8_on(p_state->buf);
	}
    }
    return;
}