#

package HTML::WebMake::WmkFile;


use HTML::WebMake::File;
use HTML::WebMake::MetaTable;
use Carp;
use strict;

use vars	qw{
  	@ISA
	$CGI_EDIT_AS_WMKFILE
	$CGI_EDIT_AS_DIR
	$CGI_EDIT_AS_TEXT
	$CGI_NON_EDITABLE
};

@ISA = qw(HTML::WebMake::File);

$CGI_EDIT_AS_WMKFILE		= 1;
$CGI_EDIT_AS_DIR		= 2;
$CGI_EDIT_AS_TEXT		= 3;
$CGI_NON_EDITABLE		= 4;

###########################################################################

sub new ($$$) {
  my $class = shift;
  $class = ref($class) || $class;
  my ($main, $filename) = @_;
  my $self = $class->SUPER::new ($main, $filename);

  $self->{cgi} = {
    'fulltext'		=> undef,
    'items'		=> [ ],
    keep_cgi_fulltext	=> 0,
    skip_eval_code	=> 0,
    skip_subst_attrs	=> 0,
    mk_cgi_items_array	=> 0,
  };

  bless ($self, $class);
  $self;
}

# -------------------------------------------------------------------------

sub dbg { HTML::WebMake::Main::dbg (@_); }
sub dbg2 { HTML::WebMake::Main::dbg2 (@_); }

# -------------------------------------------------------------------------

sub parse {
  my ($self, $str) = @_;
  local ($_) = $str;

  if (!defined $self->{main}) { carp "no main defined in WmkFile::parse"; }

  if ($self->{keep_cgi_fulltext}) {
    $self->{cgi}->{fulltext} = $_;
  }

  # We don't use a proper XML parser, because:
  # (a) content blocks etc. can contain HTML tags which will not be
  # scoped correctly;
  # (b) we use <{perl }> blocks which are invalid XML;
  # (c) we allow attributes without "quotes".
  # So kludge it where required.  We're probably faster this way
  # anyway ;)

  # trim off text before/after <webmake> chunk
  s/^.*?<webmake\b[^>]*?>//gis;
  s/<\/\s*webmake\s*>.*$//gis;

  $self->{scopings} = { };
  for my $tag (qw(for metadefault attrdefault usemetaset)) {
    $self->{scopings}->{$tag} = 0;
  }
  $self->fix_scoped_tags (\$_);

  my $util = $self->{main}->{util};
  if (!defined $util) { carp "no util defined in WmkFile::parse"; }

  $util->set_filename ($self->{filename});

  # if we are parsing for the CGI scripts, make sure that the XML
  # parser also notes regular expressions which match each item, so that the
  # CGI code can rewrite the file easily later.
  if ($self->{mk_cgi_items_array}) {
    $util->{generate_tag_regexps} = 1;
  }

  my $prevpass;
  my ($lasttag, $lasteval);
  for (my $evalpass = 0; 1; $evalpass++) {
    last if (defined $prevpass && $_ eq $prevpass);
    $prevpass = $_;

    s/^\s+//gs;
    last if ($_ !~ /^</);

    1 while s/<\{!--.*?--\}>//gs;	# WebMake comments.
    1 while s/^<!--.*?-->//gs;		# XML-style comments.

    # Preprocessing.
    if (!$self->{skip_eval_code}) {
      $self->{main}->eval_code_at_parse (\$_);

      # This may have been set by a plugin that needs introspection
      # into the .wmk file; check it here.
      if ($self->{main}->{start_parsing_introspection}) {
	$self->set_parse_introspection();
	$util->{generate_tag_regexps} = 1;
        $self->{main}->{start_parsing_introspection} = 0;
      }

    } else {
      1 while s/^<{.*?}>//gs;		# trim code, CGI mode doesn't need it
    }

    $self->{main}->getusertags()->subst_wmk_tags
				      ($self->{filename}, \$_);
     
    {
      # if we got some eval code, store the text for error messages
      my $text = $self->{main}->{last_perl_code_text};
      if (defined $text) { $lasteval = $text; $lasttag = undef; }
    }

    # Tags, ordered from most-likely to least-likely...
    $util->strip_first_tag_block (\$_, "out",
				  $self, \&tag_out, qw(file));
    $util->strip_first_tag_block (\$_, "content",
				  $self, \&tag_content, qw(name));
    $util->strip_first_tag_block (\$_, "template",
				  $self, \&tag_template, qw(name));
    $util->strip_first_lone_tag (\$_, "include",
				  $self, \&tag_include, qw(file));
    $util->strip_first_lone_tag (\$_, "use",
				  $self, \&tag_use, qw(plugin));
    $util->strip_first_lone_tag (\$_, "contents",
				  $self, \&tag_contents, qw(src name));
    $util->strip_first_lone_tag (\$_, "templates",
				  $self, \&tag_templates, qw(src name));
    $util->strip_first_lone_tag (\$_, "media",
				  $self, \&tag_media, qw(src name));
    $util->strip_first_tag_block (\$_, "contenttable",
				  $self, \&tag_contenttable, qw());

    if (/^<metadefault/i) {
      $util->strip_first_lone_tag (\$_, "metadefault",
				    $self, \&tag_metadefault, qw(name));
      my $i;
      for ($i = 0; $i < $self->{scopings}->{"metadefault"}; $i++) {
	$util->strip_first_tag_block (\$_, "metadefault".$i,
				    $self, \&tag_metadefault, qw(name));
      }
    }
    if (/^<attrdefault/i) {
      $util->strip_first_lone_tag (\$_, "attrdefault",
				    $self, \&tag_attrdefault, qw(name));
      my $i;
      for ($i = 0; $i < $self->{scopings}->{"attrdefault"}; $i++) {
	$util->strip_first_tag_block (\$_, "attrdefault".$i,
				    $self, \&tag_attrdefault, qw(name));
      }
    }
    if (/^<usemetaset/i) {
      my $i;
      for ($i = 0; $i < $self->{scopings}->{"usemetaset"}; $i++) {
	$util->strip_first_tag_block (\$_, "usemetaset".$i,
				    $self, \&tag_usemetaset, qw(id));
      }
    }

    $util->strip_first_tag (\$_, "metatable",
				  $self, \&tag_metatable, qw());
    $util->strip_first_tag (\$_, "metaset",
				  $self, \&tag_metaset, qw(id));
    $util->strip_first_tag (\$_, "sitemap",
				  $self, \&tag_sitemap, qw(name node leaf));
    $util->strip_first_tag (\$_, "navlinks",
				  $self, \&tag_navlinks,
				  qw(name map up prev next));
    $util->strip_first_lone_tag (\$_, "breadcrumbs",
				  $self, \&tag_breadcrumbs,
				  qw(name map level));

    # Loops
    if (/^<for/i) {
      my $i;
      for ($i = 0; $i < $self->{scopings}->{"for"}; $i++) {
	$util->strip_first_tag_block (\$_, "for".$i,
				      $self, \&tag_for, qw(name values));
      }
    }

    # Misc.
    $util->strip_first_lone_tag (\$_, "cache",
				  $self, \&tag_cache, qw(dir));
    $util->strip_first_lone_tag (\$_, "option",
				  $self, \&tag_option, qw(name value));

    # CGIs and hrefs
    $util->strip_first_lone_tag (\$_, "editcgi",
				  $self, \&tag_editcgi, qw(href));
    $util->strip_first_lone_tag (\$_, "viewcgi",
				  $self, \&tag_viewcgi, qw(href));
    $util->strip_first_lone_tag (\$_, "site",
				  $self, \&tag_site, qw(href));
    $util->strip_first_tag_block (\$_, "action",
				  $self, \&tag_action, qw(event));

    # if we got some tags, store the text for error messages
    my $text = $util->{last_tag_text};
    if (defined $text) { $lasttag = $text; $lasteval = undef; }
  }

  # if there's any text left in the file that we couldn't parse,
  # it's an error, so warn about it.
  #
  if (/\S/) {
    my $failuretext = $lasttag;

    if (defined $lasteval) {
      if ($_ !~ /^</) {
	# easy to spot; the Perl code returned '1' or something.
	# flag it clearly.

	s/\n.*$//gs;
	$self->{main}->fail ("Perl code didn't return valid WebMake code:\n".
		"\t$lasteval\n\t=> \"$_\"\n");
	return 0;
      }
      $failuretext = $lasteval;
    }

    /^([^<].*?>.{40,40})/s; if (defined $1) { $_ = $1; }
    s/\s+/ /gs;
    $lasttag ||= '';
    $self->{main}->fail ("WMK file contains unparseable data at or after:\n".
	      "\t$lasttag\n\t$_ ...\"\n");
    return 0;
  }

  return 1;
}

# -------------------------------------------------------------------------

# handle scoped tags.  Since we don't use a proper XML parser, we have to
# rewrite them here.  We convert them to single-character markers (\001 or
# \002) indicating a start tag or end tag, then loop until all appearances of
# the tag have been converted. We then convert them back to text, with a
# scope number attached.  Until Perl can do a regexp like this:
#
# /<tag[^>]*>[^<tag]+<\/tag>/
#
# we're probably stuck doing it this way.  Hey, don't knock it, it works ;)

sub fix_scoped_tags {
  my ($self, $txt) = @_;

  $$txt =~ s/\001/<<001>>/gs;
  $$txt =~ s/\002/<<002>>/gs;

  for my $tag (qw(for metadefault attrdefault usemetaset)) {
    next if ($$txt !~ /<\/$tag>/);

    $$txt =~ s/<$tag(\b[^>]*[^\/]>)/\001$1/gs;
    $$txt =~ s/<\/$tag>/\002/gs;

    my $count = $self->{scopings}->{$tag};
    while ($$txt =~ s{\001([^>]+)>([^\001\002]+)\002}
    			{<$tag$count$1>$2<\/$tag$count>}gis)
    {
      $self->{scopings}->{$tag}++;
      $count++;
    }
  }
  $$txt =~ s/<<001>>/\001/gs;
  $$txt =~ s/<<002>>/\002/gs;
}

# -------------------------------------------------------------------------

sub subst_attrs {
  my ($self, $tagname, $attrs) = @_;
  return if ($self->{skip_subst_attrs});

  if (defined ($attrs->{name})) {
    $tagname .= " \"".$attrs->{name}."\"";	# for errors
  }

  my ($k, $v);
  while (($k, $v) = each %{$attrs}) {
    next unless (defined $k && defined $v);
    $attrs->{$k} = $self->{main}->fileless_subst ($tagname, $v);
  }
}

# -------------------------------------------------------------------------

sub tag_include {
  my ($self, $tag, $attrs, $text) = @_;

  $self->cgi_add ($tag, $CGI_EDIT_AS_WMKFILE, $attrs->{file}, $attrs) and return '';
  $self->subst_attrs ("<include>", $attrs);

  my $file = $attrs->{file};

  if (!open (INC, "< $file")) {
    die "Cannot open include file: $file\n";
  }
  my @s = stat INC;
  my $inc = join ('', <INC>);
  close INC;

  dbg ("included file: \"$file\"");
  $self->{main}->set_file_modtime ($file, $s[9]);
  $self->add_dep ($file);

  $self->fix_scoped_tags (\$inc);
  $inc;
}

# -------------------------------------------------------------------------

sub tag_use {
  my ($self, $tag, $attrs, $text) = @_;

  $self->subst_attrs ("<use>", $attrs);

  my $plugin = $attrs->{plugin};
  my $file;
  my @s;

  $file = '~/.webmake/plugins/'.$plugin.'.wmk';
  $file = $self->{main}->sed_fname ($file);
  @s = stat $file;

  if (!defined $s[9]) {
    $file = '%l/'.$plugin.'.wmk';
    $file = $self->{main}->sed_fname ($file);
    @s = stat $file;
  }

  if (!defined $s[9]) {
    die "Cannot open 'use' plugin: $plugin\n";
  }

foundit:
  if (!open (INC, "<$file")) {
    die "Cannot open 'use' file: $file\n";
  }
  my $inc = join ('', <INC>);
  close INC;

  dbg ("used file: \"$file\"");
  $self->{main}->set_file_modtime ($file, $s[9]);
  $self->add_dep ($file);

  $self->fix_scoped_tags (\$inc);

  # make the <use> tag's attributes available to the plugin
  my $attrstr = '';
  foreach my $key (keys %{$attrs}) {
    next if ($key eq 'plugin');
    my $val = $attrs->{$key};
    $val =~ s!</template>!\&lt;/template>!g;	# just in case
    $attrstr .= "<template name=\"$plugin.$key\">$val</template>\n";
  }

  $attrstr.$inc;
}

# -------------------------------------------------------------------------

sub tag_cache {
  my ($self, $tag, $attrs, $text) = @_;

  $self->subst_attrs ("<cache>", $attrs);
  my $dir = $attrs->{dir};
  $self->{main}->setcachefile ($dir);
  "";
}

# -------------------------------------------------------------------------

sub tag_option {
  my ($self, $tag, $attrs, $text) = @_;

  $self->subst_attrs ("<option>", $attrs);
  $self->{main}->set_option ($attrs->{name}, $attrs->{value});
  "";
}

# -------------------------------------------------------------------------

sub tag_editcgi {
  my ($self, $tag, $attrs, $text) = @_;

  $self->subst_attrs ("<editcgi>", $attrs);
  my $wmkf = 'main.wmk';		# TODO

  $self->{main}->add_url ("WebMake.EditCGI", $attrs->{href}
  		."?wmkf=".$wmkf);
  "";
}

# -------------------------------------------------------------------------

sub tag_viewcgi {
  my ($self, $tag, $attrs, $text) = @_;

  $self->subst_attrs ("<viewcgi>", $attrs);
  $self->{main}->add_url ("WebMake.ViewCGI", $attrs->{href});
  "";
}

# -------------------------------------------------------------------------

sub tag_action {
  my ($self, $tag, $attrs, $text) = @_;

  $self->subst_attrs ("<action>", $attrs);
  $self->{main}->add_action ($attrs->{event}, $text);
  "";
}

# -------------------------------------------------------------------------

sub tag_site {
  my ($self, $tag, $attrs, $text) = @_;

  $self->subst_attrs ("<site>", $attrs);
  $self->{main}->add_url ("WebMake.SiteHref", $attrs->{href});
  "";
}

# -------------------------------------------------------------------------

sub tag_content {
  my ($self, $tag, $attrs, $text) = @_;

  $self->cgi_add ($tag, $CGI_EDIT_AS_TEXT, $text, $attrs) and return '';
  $self->subst_attrs ("<content>", $attrs);
  my $name = $attrs->{name};
  if (!defined $name) {
    carp ("Unnamed content found in ".$self->{filename}.": $text\n");
    return;
  }

  if (defined $attrs->{root}) {
    warn "warning: \${$name}: 'root' attribute is deprecated, ".
    		"use 'isroot' instead\n";
    $attrs->{isroot} = $attrs->{root};	# backwards compat
  }

  if (defined $attrs->{src}) {
    $text = $self->read_src ('content', $attrs, $text);
  }

  $self->{main}->add_content ($name, $self, $attrs, $text);
  "";
}

sub tag_contents {
  my ($self, $tag, $attrs, $text) = @_;

  $self->cgi_add_datasource ($tag, $attrs) and return '';
  $self->subst_attrs ("<contents>", $attrs);
  my $lister = new HTML::WebMake::Contents ($self->{main},
  			$attrs->{src}, $attrs->{name}, $attrs);
  $lister->add();
  "";
}

sub tag_template {
  my ($self, $tag, $attrs, $text) = @_;

  $self->cgi_add ($tag, $CGI_EDIT_AS_TEXT, $text, $attrs) and return '';
  $self->subst_attrs ("<template>", $attrs);
  my $name = $attrs->{name};
  if (!defined $name) {
    carp ("Unnamed template found in ".$self->{filename}.": $text\n");
    return;
  }
  $attrs->{map} = 'false';

  if (defined $attrs->{src}) {
    $text = $self->read_src ('template', $attrs, $text);
  }

  $self->{main}->add_content ($name, $self, $attrs, $text);
  "";
}

sub tag_templates {
  my ($self, $tag, $attrs, $text) = @_;

  $self->cgi_add_datasource ($tag, $attrs) and return '';
  $self->subst_attrs ("<templates>", $attrs);
  $attrs->{map} = 'false';
  my $lister = new HTML::WebMake::Contents ($self->{main},
  			$attrs->{src}, $attrs->{name}, $attrs);
  $lister->add();
  "";
}

sub tag_media {
  my ($self, $tag, $attrs, $text) = @_;

  $self->cgi_add_datasource ($tag, $attrs) and return '';
  $self->subst_attrs ("<media>", $attrs);
  my $lister = new HTML::WebMake::Media ($self->{main},
  			$attrs->{src}, $attrs->{name}, $attrs);
  $lister->add();
  "";
}

sub tag_contenttable {
  my ($self, $tag, $attrs, $text) = @_;

  $self->cgi_add ($tag, $CGI_EDIT_AS_TEXT, $text, $attrs) and return '';
  $self->subst_attrs ("<contenttable>", $attrs);

  if (defined $attrs->{src}) {
    $text = $self->read_src ('contenttable', $attrs, $text);
  }

  # we actually use a Contents object, reading from the .wmk file
  # to do this.
  $attrs->{src} = 'svfile:';
  if (!defined $attrs->{name})		{ $attrs->{name} = '*'; }
  if (!defined $attrs->{namefield})	{ $attrs->{namefield} = '1'; }
  if (!defined $attrs->{valuefield})	{ $attrs->{valuefield} = '2'; }

  my $lister = new HTML::WebMake::Contents ($self->{main},
  			$attrs->{src}, $attrs->{name}, $attrs);

  $lister->{ctable_wmkfile} = $self;
  $lister->{ctable_text} = $text;

  $lister->add();
  "";
}

sub tag_metadefault {
  my ($self, $tag, $attrs, $text) = @_;

  $self->cgi_add ($tag, $CGI_NON_EDITABLE, undef, $attrs) and return $text;
  $self->subst_attrs ("<metadefault>", $attrs);
  $self->{main}->{metadata}->set_metadefault ($attrs->{name}, $attrs->{value});

  return '' if (!defined $text || $text eq '');
  $text . '<metadefault name="'.$attrs->{name}.'" value="[POP]" />';
}

sub tag_attrdefault {
  my ($self, $tag, $attrs, $text) = @_;

  $self->cgi_add ($tag, $CGI_NON_EDITABLE, undef, $attrs) and return $text;
  $self->subst_attrs ("<attrdefault>", $attrs);
  $self->{main}->{metadata}->set_attrdefault ($attrs->{name}, $attrs->{value});

  return '' if (!defined $text || $text eq '');
  $text . '<attrdefault name="'.$attrs->{name}.'" value="[POP]" />';
}

sub tag_usemetaset {
  my ($self, $tag, $attrs, $text) = @_;

  $self->cgi_add ($tag, $CGI_NON_EDITABLE, undef, $attrs) and return $text;
  $self->subst_attrs ("<usemetaset>", $attrs);

  # set it on Main in case perl code accesses metadata (quite likely!)
  $self->{main}->{metaset} = $attrs->{id};

  # also set it on Content items etc.; sneaky kludge!
  # Use 'attrdefault' to implement this
  $self->{main}->{metadata}->set_attrdefault ('usemetaset', $attrs->{id});

  return '' if (!defined $text || $text eq '');
  $text . '<attrdefault name="usemetaset" value="[POP]" />';
}

sub tag_metaset {
  my ($self, $tag, $attrs, $text) = @_;

  $self->cgi_add ($tag, $CGI_NON_EDITABLE, undef, $attrs) and return $text;
  $self->subst_attrs ("<metaset>", $attrs);
  $self->{main}->{metadata}->parse_metaset ($attrs->{id}, $text, $attrs);
  '';
}

sub tag_metatable {
  my ($self, $tag, $attrs, $text) = @_;

  $self->cgi_add ($tag, $CGI_EDIT_AS_TEXT, $text, $attrs) and return '';
  $self->subst_attrs ("<metatable>", $attrs);

  if (defined $attrs->{src}) {
    $text = $self->read_src ('metatable', $attrs, $text);
  }

  my $tbl = new HTML::WebMake::MetaTable ($self->{main});
  $tbl->parse_metatable ($attrs, $text);
  "";
}

sub tag_sitemap {
  my ($self, $tag, $attrs, $text) = @_;

  $self->cgi_add ($tag, $CGI_EDIT_AS_TEXT, $text, $attrs) and return '';
  $self->subst_attrs ("<sitemap>", $attrs);
  $self->{main}->add_sitemap ($attrs->{name},
  			$attrs->{rootname}, $self, $attrs, $text);
  "";
}

sub tag_navlinks {
  my ($self, $tag, $attrs, $text) = @_;

  $self->cgi_add ($tag, $CGI_EDIT_AS_TEXT, $text, $attrs) and return '';
  $self->subst_attrs ("<navlinks>", $attrs);
  $self->{main}->add_navlinks ($attrs->{name}, $attrs->{map},
  			$self, $attrs, $text);
  "";
}

sub tag_breadcrumbs {
  my ($self, $tag, $attrs, $text) = @_;

  $self->cgi_add ($tag, $CGI_EDIT_AS_TEXT, $text, $attrs) and return '';
  $self->subst_attrs ("<breadcrumbs>", $attrs);
  $attrs->{top} ||= $attrs->{level};
  $attrs->{tail} ||= $attrs->{level};
  $self->{main}->add_breadcrumbs ($attrs->{name}, $attrs->{map},
  			$self, $attrs, $text);
  "";
}

sub tag_out {
  my ($self, $tag, $attrs, $text) = @_;

  $self->cgi_add ($tag, $CGI_EDIT_AS_TEXT, $text, $attrs) and return '';
  $self->subst_attrs ("<out>", $attrs);
  my $file = $attrs->{file};
  my $name = $attrs->{name}; $name ||= $file;

  $self->{main}->add_out ($file, $self, $name, $attrs, $text);

  my $lang = $self->{main}->{outs}->{$file}->{lang};
  $name = $name."::".$lang if defined $lang;

  $self->{main}->add_url ($name, $file);
  "";
}

sub tag_for ($$$$) {
  my ($self, $tag, $attrs, $text) = @_;
  local ($_);

  $self->cgi_add ($tag, $CGI_NON_EDITABLE, undef, $attrs) and return $text;
  $self->subst_attrs ("<for>", $attrs);

  my $name = $attrs->{name};
  my $namesubst = $attrs->{namesubst};
  my $vals = $attrs->{'values'};

  my @vals = split (' ', $vals);
  if ($#vals >= 0)
  {
    if (!$self->{main}->{paranoid}) {
      if (defined $namesubst) {
	eval '
	  @vals = map { '.$namesubst.'; $_; } @vals;
	';
      }
      if ($#vals < 0 || $@) {
	warn ("<for> tag \"$attrs->{name}\" namesubst failed: $@\n");
      }

    } else {
      warn "Paranoid mode on: not processing namesubst\n";
    }
  }

  my $ret = '';
  foreach my $val (@vals) {
    next if (!defined $val || $val eq '');
    $_ = $text; s/\$\{${name}\}/${val}/gs;
    $ret .= $_;
  }

  dbg2 ("for tag evaluated: \"$ret\"");
  $ret;
}

###########################################################################

sub cgi_add {
  my ($self, $tag, $editui, $edituidata, $attrs) = @_;

  return undef unless ($self->{mk_cgi_items_array});

  my $name = "$tag";
  if (defined $attrs->{name}) {
    $name = "$tag name=\"".$attrs->{name}."\"";
  }

  my $re = $self->{main}->{util}->{last_tag_regexp};

  my $id = $re;
  $id =~ tr/=/E/;
  $id =~ s/[\\<>\'\"]//gs;
  $id =~ s/[^-_A-Za-z0-9]+/_/gs;
  $id =~ s/^_six-m_//; $id =~ s/_$//;

  my $item = {
    'tag'		=> $tag,
    'name'		=> $name,
    'attrs'		=> $attrs,
    'id'		=> $id,
    'editui'		=> $editui,
    'edituidata'	=> $edituidata,
    'origtagregexp'	=> $re,
  };

  push (@{$self->{cgi}->{items}}, $item);

  if (!$self->{skip_adding_items}) { return undef; }
  return ' ';
}

sub cgi_add_datasource {
  my ($self, $tag, $attrs) = @_;

  return undef unless ($self->{mk_cgi_items_array});

  my $proto = 'file';
  my $src = $attrs->{src};
  if ($src =~ s/^([A-Za-z0-9]+)://) {
    $proto = $1; $proto =~ tr/A-Z/a-z/;
  }

  if ($proto eq 'file') {
    $self->cgi_add ($tag, $CGI_EDIT_AS_DIR, $src, $attrs);
  }

  if (!$self->{skip_adding_items}) { return undef; }
  return ' ';
}

###########################################################################

sub read_src {
  my ($self, $tag, $attrs, $oldtext) = @_;

  my $src = $attrs->{src};
  my $text;
  my $file;

  if ($src =~ /^(http|https|ftp):/) {
    my $updfreq = $attrs->{updatefreq};
    $updfreq ||= "1h";			# 1 hour by default
    $updfreq = $self->{main}->{util}->timespec_to_seconds ($updfreq);

    dbg ("attempting to load \"$src\" using LWP...");

    # using LWP::Simple::mirror, we can cache the content to disk
    # and only refetch it when it changes.
    #
    my $tmpdir = File::Spec->catdir ($self->{main}->tmpdir(), "srcs");
    if (! -d $tmpdir) { mkdir ($tmpdir, 0777); }

    my $tmpfile = $src;
    $tmpfile =~ s/[^-_=\.\,\+A-Za-z0-9]+/_/gs;
    $tmpfile = File::Spec->catdir ($tmpdir, $tmpfile);

    my $rcode;

    if (-f $tmpfile && (-M $tmpfile) * 24*60*60 < $updfreq) {
      dbg ("cached copy is young enough, not refetching");
      $file = $tmpfile;

    } else {
      eval {
	  require LWP::Simple;
	  $rcode = LWP::Simple::mirror ($src, $tmpfile);
      };

      if ($@) {
	warn "HTTP get $src failed: LWP module not available\n";
      } elsif (LWP::Simple::is_error ($rcode)) {
	warn "HTTP get $src failed: HTTP error code $rcode.\n";
      } else {
	$file = $tmpfile;
      }
    }

  } else {
    $file = $src;
  }

  if (defined $file) {
    $file =~ s/^file:\/+/\//g;	# remove optional file: prefix
    if (open (IN, "<".$file)) {
      binmode IN;		# in case it's an image etc.
      $text = '';
      while (sysread IN, $_, 4096) { $text .= $_; }
      close IN;

      dbg ("$file: sum=".unpack ("%32C*", $text)." len=".length($text));
    }
  }

  if (!defined $text) {
    warn ("<$tag src=\"$attrs->{src}\"> could not be read.\n");
    return '';
  }

  return $text;
}

###########################################################################

# CGI-only introspection: inhibits code parsing and attr substs
sub set_parse_for_cgi {
  my $self = shift;
  $self->{keep_cgi_fulltext} = 1;
  $self->{mk_cgi_items_array} = 1;
  $self->{skip_adding_items} = 1;
  $self->{skip_eval_code} = 1;
  $self->{skip_subst_attrs} = 1;
}

# lightweight introspection: do not inhibit normal parsing.
sub set_parse_introspection {
  my $self = shift;
  $self->{keep_cgi_fulltext} = 1;
  $self->{mk_cgi_items_array} = 1;
}

1;