One more follow up for anyone that might find this useful. I modified the struct a bit so it reflects 'path' rather than 'filename' and I added a string for encoding ('enc') so I can mark when a file is gzipped or not. I went back to perl for writing the script to generate the header file as there were existing minifier libraries on cpan (although not the greatest I've see - I may play around with the node.js command line minifier code)
The perl script is crude (I used to be a perl expert but haven't used in in more than 6 years presently) and relised on the HTML, CSS and JavaScript Packer libraries for minifying those three file types. After minifying I use Faster::Gzip to compress them. I also compress down anything that shows up at text/plain.
I decided not to do anything fancy with the mime type detection and it instead expects you are using standard file extensions and determines the mime type from the file name extension using MIME::Types. File reads/writes are done with File::Slurp calls with the exception of the streaming input from the xxd command (the perl xxd library didn't have the -i formatting and I didn't find anything else right away that was as clean and suitable to the c++ needs)
I ended up doing the file collection with a simple streaming open() handle to the unix find command. Syntax should be similar in cygwin. It seeks out files specifically in the $webdir folder. A match is any file with a a standard 1+ character filename with a 2-4 character extension after the '.'
The sub folder name is chopped off of the find results to create the resulting path used by the code on the esp.
hexify.pl:
Code: Select all#!/usr/bin/perl
use CSS::Packer;
use JavaScript::Packer;
use HTML::Packer;
use File::Slurp;
use Gzip::Faster;
use MIME::Types;
my $outfile = "webfiles.h";
my $webdir = "www";
my $find_exe = "/usr/bin/find";
my $xxdexe = "/usr/bin/xxd -i";
my $findcmd = $find_exe . " " . $webdir . " -type f ";
my $mt = MIME::Types->new();
my $files = open(FINDSTRM, "$findcmd|") || die $!;
my @filelist;
while($line = <FINDSTRM>) {
chomp($line);
if($line =~ /^$webdir(.*)\/([^\.\/]+)\.(\w{2,4})$/) {
my %fileinfo;
$fileinfo{path} = "$1/$2.$3";
$fileinfo{dir} = $1;
$fileinfo{name} = $2;
$fileinfo{ext} = $3;
my $type = $mt->mimeTypeOf($fileinfo{ext});
$fileinfo{type} = $type->{MT_type};
my $bindat;
if($type->{MT_type} eq "text/html") {
print "html minifying $line\n";
my $html = read_file($line);
my $packer = HTML::Packer->init();
my $htmin = $packer->minify( \$html, { remove_newlines => 1, remove_comments => 1, do_javascript => 'best', do_stylesheet => 'minify' } );
$bindat = gzip($htmin);
}
if($type->{MT_type} eq "text/css") {
print "css minifying $line\n";
my $css = read_file($line);
my $packer = CSS::Packer->init();
my $cssmin = $packer->minify(\$css, { compress => 'minify', remove_comments => 1 } );
$bindat = gzip($cssmin);
}
if($type->{MT_type} eq "application/javascript") {
print "javascript minifying $line\n";
my $js = read_file($line);
my $packer = JavaScript::Packer->init();
my $jsmin = $packer->minify(\$js, { compress => 'best', remove_comments => 1, remove_copyright => 1 } );
$bindat = gzip($jsmin);
}
# other text based formats
if( $type->{MT_type} eq "text/plain" ) {
print "compressing text in $line\n";
my $asc = read_file($line);
$bindat = gzip($asc);
}
if(length($bindat) == 0) {
print "raw reading $line\n";
$bindat = read_file($line, { binmode => ':raw' });
$fileinfo{enc} = '';
} else {
$fileinfo{enc} = 'gzip';
}
{
use bytes;
$fileinfo{len} = length($bindat);
}
$tfile = "/tmp/tmpfile";
write_file( $tfile, {binmode => ':raw'}, $bindat );
open(INXXD, "$xxdexe < $tfile |") || die $!;
$fileinfo{hexdat} = do { local $/; <INXXD> };
close INXXD;
$fileinfo{hexdat} =~ s/^\s+|\s+$//g;
push(@filelist,\%fileinfo);
}
}
close FINDSTRM;
open(OUTFILE, ">", $outfile) || die $!;
print OUTFILE "\n#include \"Flash.h\"\n\n";
$count = 1;
foreach $f (@filelist) {
print OUTFILE "FLASH_ARRAY(uint8_t, file_" . $count . ",\n";
print OUTFILE " " . $f->{hexdat};
print OUTFILE ");\n\n";
$count++;
}
print OUTFILE "struct t_websitefiles {\n";
print OUTFILE " const char* path;\n";
print OUTFILE " const char* mime;\n";
print OUTFILE " const unsigned int len;\n";
print OUTFILE " const char* enc;\n";
print OUTFILE " const _FLASH_ARRAY<uint8_t>* content;\n";
print OUTFILE "} files[] = {\n";
$count = 1;
foreach $f (@filelist) {
print OUTFILE " {\n";
print OUTFILE " .path = \"" . $f->{path} . "\",\n";
print OUTFILE " .mime = \"" . $f->{type} . "\",\n";
print OUTFILE " .len = " . $f->{len} . ",\n";
print OUTFILE " .enc = \"" . $f->{enc} . "\",\n";
print OUTFILE " .content = \&file_" . $count . ",\n";
print OUTFILE " },\n";
$count++;
}
print OUTFILE "};\n";
close OUTFILE;
Since I changed the one variable from filename to path in the struct and added the enc type, there are a couple other mods to the c++ loadFromFlash function also:
Code: Select allbool loadFromFlash(String path) {
if(path.endsWith("/")) path += "index.htm";
int NumFiles = sizeof(files)/sizeof(struct t_websitefiles);
for(int i=0; i<NumFiles; i++) {
if(path == String(files[i].path)) {
_FLASH_ARRAY<uint8_t>* filecontent;
String dataType = "text/plain";
unsigned int len = 0;
dataType = files[i].mime;
len = files[i].len;
if(files[i].enc != "")
webServer.sendHeader("Content-Encoding", files[i].enc);
webServer.setContentLength(len);
webServer.send(200, files[i].mime, "");
filecontent = (_FLASH_ARRAY<uint8_t>*)files[i].content;
filecontent->open();
WiFiClient client = webServer.client();
client.write(*filecontent, 100);
return true;
}
}
return false;
}