Parsing html entities in the URLs so we get the right hash

2 files changed, 490 insertions, 1 deletions

diff --git a/httracker b/httracker
index 7fc33c1..59fcfa9 100755
--- a/httracker
+++ b/httracker
@@ -8,7 +8,8 @@ source `dirname $0`/lib/httracker/functions || exit 1
 
 # Get URLs from feed
 # Thanks http://stackoverflow.com/questions/443991/how-to-parse-rss-feeds-xml-in-a-shell-script
-curl -s "$FEED" | grep -o '<link>[^<]*' | grep -o "[^>]*$" > $URLS
+curl -s "$FEED" | grep -o '<link>[^<]*' | grep -o "[^>]*$" \
+     | `dirname $0`/lib/httracker/html.sed > $URLS
 
 if [ "$?" != "0" ]; then
   echo "Error downloading feed $FEED, aborting."
diff --git a/lib/httracker/html.sed b/lib/httracker/html.sed
new file mode 100755
index 0000000..a135ca6
--- /dev/null
+++ b/lib/httracker/html.sed
@@ -0,0 +1,488 @@
+#!/bin/sed -f
+# Thanks https://gist.github.com/mlk/5222903/raw/6ef14f5fc351ed057d97da2fe1d83dcefc64c817/html.sed
+s/&#160;/ /g
+s/&nbsp;/ /g
+s/&#161;/¡/g
+s/&iexcl;/¡/g
+s/&#162;/¢/g
+s/&cent;/¢/g
+s/&#163;/£/g
+s/&pound;/£/g
+s/&#164;/¤/g
+s/&curren;/¤/g
+s/&#165;/¥/g
+s/&yen;/¥/g
+s/&#166;/¦/g
+s/&brvbar;/¦/g
+s/&#167;/§/g
+s/&sect;/§/g
+s/&#168;/¨/g
+s/&uml;/¨/g
+s/&#169;/©/g
+s/&copy;/©/g
+s/&#170;/ª/g
+s/&ordf;/ª/g
+s/&#171;/«/g
+s/&laquo;/«/g
+s/&#172;/¬/g
+s/&not;/¬/g
+s/&#173;/­­/g
+s/&shy;/­­/g
+s/&#174;/®/g
+s/&reg;/®/g
+s/&#175;/¯/g
+s/&macr;/¯/g
+s/&#176;/°/g
+s/&deg;/°/g
+s/&#177;/±/g
+s/&plusmn;/±/g
+s/&#178;/²/g
+s/&sup2;/²/g
+s/&#179;/³/g
+s/&sup3;/³/g
+s/&#180;/´/g
+s/&acute;/´/g
+s/&#181;/µ/g
+s/&micro;/µ/g
+s/&#182;/¶/g
+s/&para;/¶/g
+s/&#183;/·/g
+s/&middot;/·/g
+s/&#184;/¸/g
+s/&cedil;/¸/g
+s/&#185;/¹/g
+s/&sup1;/¹/g
+s/&#186;/º/g
+s/&ordm;/º/g
+s/&#187;/»/g
+s/&raquo;/»/g
+s/&#188;/¼/g
+s/&frac14;/¼/g
+s/&#189;/½/g
+s/&frac12;/½/g
+s/&#190;/¾/g
+s/&frac34;/¾/g
+s/&#191;/¿/g
+s/&iquest;/¿/g
+s/&#215;/×/g
+s/&times;/×/g
+s/&#247;/÷/g
+s/&divide;/÷/g
+s/&#192;/À/g
+s/&Agrave;/À/g
+s/&#193;/Á/g
+s/&Aacute;/Á/g
+s/&#194;/Â/g
+s/&Acirc;/Â/g
+s/&#195;/Ã/g
+s/&Atilde;/Ã/g
+s/&#196;/Ä/g
+s/&Auml;/Ä/g
+s/&#197;/Å/g
+s/&Aring;/Å/g
+s/&#198;/Æ/g
+s/&AElig;/Æ/g
+s/&#199;/Ç/g
+s/&Ccedil;/Ç/g
+s/&#200;/È/g
+s/&Egrave;/È/g
+s/&#201;/É/g
+s/&Eacute;/É/g
+s/&#202;/Ê/g
+s/&Ecirc;/Ê/g
+s/&#203;/Ë/g
+s/&Euml;/Ë/g
+s/&#204;/Ì/g
+s/&Igrave;/Ì/g
+s/&#205;/Í/g
+s/&Iacute;/Í/g
+s/&#206;/Î/g
+s/&Icirc;/Î/g
+s/&#207;/Ï/g
+s/&Iuml;/Ï/g
+s/&#208;/Ð/g
+s/&ETH;/Ð/g
+s/&#209;/Ñ/g
+s/&Ntilde;/Ñ/g
+s/&#210;/Ò/g
+s/&Ograve;/Ò/g
+s/&#211;/Ó/g
+s/&Oacute;/Ó/g
+s/&#212;/Ô/g
+s/&Ocirc;/Ô/g
+s/&#213;/Õ/g
+s/&Otilde;/Õ/g
+s/&#214;/Ö/g
+s/&Ouml;/Ö/g
+s/&#216;/Ø/g
+s/&Oslash;/Ø/g
+s/&#217;/Ù/g
+s/&Ugrave;/Ù/g
+s/&#218;/Ú/g
+s/&Uacute;/Ú/g
+s/&#219;/Û/g
+s/&Ucirc;/Û/g
+s/&#220;/Ü/g
+s/&Uuml;/Ü/g
+s/&#221;/Ý/g
+s/&Yacute;/Ý/g
+s/&#222;/Þ/g
+s/&THORN;/Þ/g
+s/&#223;/ß/g
+s/&szlig;/ß/g
+s/&#224;/à/g
+s/&agrave;/à/g
+s/&#225;/á/g
+s/&aacute;/á/g
+s/&#226;/â/g
+s/&acirc;/â/g
+s/&#227;/ã/g
+s/&atilde;/ã/g
+s/&#228;/ä/g
+s/&auml;/ä/g
+s/&#229;/å/g
+s/&aring;/å/g
+s/&#230;/æ/g
+s/&aelig;/æ/g
+s/&#231;/ç/g
+s/&ccedil;/ç/g
+s/&#232;/è/g
+s/&egrave;/è/g
+s/&#233;/é/g
+s/&eacute;/é/g
+s/&#234;/ê/g
+s/&ecirc;/ê/g
+s/&#235;/ë/g
+s/&euml;/ë/g
+s/&#236;/ì/g
+s/&igrave;/ì/g
+s/&#237;/í/g
+s/&iacute;/í/g
+s/&#238;/î/g
+s/&icirc;/î/g
+s/&#239;/ï/g
+s/&iuml;/ï/g
+s/&#240;/ð/g
+s/&eth;/ð/g
+s/&#241;/ñ/g
+s/&ntilde;/ñ/g
+s/&#242;/ò/g
+s/&ograve;/ò/g
+s/&#243;/ó/g
+s/&oacute;/ó/g
+s/&#244;/ô/g
+s/&ocirc;/ô/g
+s/&#245;/õ/g
+s/&otilde;/õ/g
+s/&#246;/ö/g
+s/&ouml;/ö/g
+s/&#248;/ø/g
+s/&oslash;/ø/g
+s/&#249;/ù/g
+s/&ugrave;/ù/g
+s/&#250;/ú/g
+s/&uacute;/ú/g
+s/&#251;/û/g
+s/&ucirc;/û/g
+s/&#252;/ü/g
+s/&uuml;/ü/g
+s/&#253;/ý/g
+s/&yacute;/ý/g
+s/&#254;/þ/g
+s/&thorn;/þ/g
+s/&#255;/ÿ/g
+s/&yuml;/ÿ/g
+s/&#8704;/∀/g
+s/&forall;/∀/g
+s/&#8706;/∂/g
+s/&part;/∂/g
+s/&#8707;/∃/g
+s/&exist;/∃/g
+s/&#8709;/∅/g
+s/&empty;/∅/g
+s/&#8711;/∇/g
+s/&nabla;/∇/g
+s/&#8712;/∈/g
+s/&isin;/∈/g
+s/&#8713;/∉/g
+s/&notin;/∉/g
+s/&#8715;/∋/g
+s/&ni;/∋/g
+s/&#8719;/∏/g
+s/&prod;/∏/g
+s/&#8721;/∑/g
+s/&sum;/∑/g
+s/&#8722;/−/g
+s/&minus;/−/g
+s/&#8727;/∗/g
+s/&lowast;/∗/g
+s/&#8730;/√/g
+s/&radic;/√/g
+s/&#8733;/∝/g
+s/&prop;/∝/g
+s/&#8734;/∞/g
+s/&infin;/∞/g
+s/&#8736;/∠/g
+s/&ang;/∠/g
+s/&#8743;/∧/g
+s/&and;/∧/g
+s/&#8744;/∨/g
+s/&or;/∨/g
+s/&#8745;/∩/g
+s/&cap;/∩/g
+s/&#8746;/∪/g
+s/&cup;/∪/g
+s/&#8747;/∫/g
+s/&int;/∫/g
+s/&#8756;/∴/g
+s/&there4;/∴/g
+s/&#8764;/∼/g
+s/&sim;/∼/g
+s/&#8773;/≅/g
+s/&cong;/≅/g
+s/&#8776;/≈/g
+s/&asymp;/≈/g
+s/&#8800;/≠/g
+s/&ne;/≠/g
+s/&#8801;/≡/g
+s/&equiv;/≡/g
+s/&#8804;/≤/g
+s/&le;/≤/g
+s/&#8805;/≥/g
+s/&ge;/≥/g
+s/&#8834;/⊂/g
+s/&sub;/⊂/g
+s/&#8835;/⊃/g
+s/&sup;/⊃/g
+s/&#8836;/⊄/g
+s/&nsub;/⊄/g
+s/&#8838;/⊆/g
+s/&sube;/⊆/g
+s/&#8839;/⊇/g
+s/&supe;/⊇/g
+s/&#8853;/⊕/g
+s/&oplus;/⊕/g
+s/&#8855;/⊗/g
+s/&otimes;/⊗/g
+s/&#8869;/⊥/g
+s/&perp;/⊥/g
+s/&#8901;/⋅/g
+s/&sdot;/⋅/g
+s/&#913;/Α/g
+s/&Alpha;/Α/g
+s/&#914;/Β/g
+s/&Beta;/Β/g
+s/&#915;/Γ/g
+s/&Gamma;/Γ/g
+s/&#916;/Δ/g
+s/&Delta;/Δ/g
+s/&#917;/Ε/g
+s/&Epsilon;/Ε/g
+s/&#918;/Ζ/g
+s/&Zeta;/Ζ/g
+s/&#919;/Η/g
+s/&Eta;/Η/g
+s/&#920;/Θ/g
+s/&Theta;/Θ/g
+s/&#921;/Ι/g
+s/&Iota;/Ι/g
+s/&#922;/Κ/g
+s/&Kappa;/Κ/g
+s/&#923;/Λ/g
+s/&Lambda;/Λ/g
+s/&#924;/Μ/g
+s/&Mu;/Μ/g
+s/&#925;/Ν/g
+s/&Nu;/Ν/g
+s/&#926;/Ξ/g
+s/&Xi;/Ξ/g
+s/&#927;/Ο/g
+s/&Omicron;/Ο/g
+s/&#928;/Π/g
+s/&Pi;/Π/g
+s/&#929;/Ρ/g
+s/&Rho;/Ρ/g
+s/&#931;/Σ/g
+s/&Sigma;/Σ/g
+s/&#932;/Τ/g
+s/&Tau;/Τ/g
+s/&#933;/Υ/g
+s/&Upsilon;/Υ/g
+s/&#934;/Φ/g
+s/&Phi;/Φ/g
+s/&#935;/Χ/g
+s/&Chi;/Χ/g
+s/&#936;/Ψ/g
+s/&Psi;/Ψ/g
+s/&#937;/Ω/g
+s/&Omega;/Ω/g
+s/&#945;/α/g
+s/&alpha;/α/g
+s/&#946;/β/g
+s/&beta;/β/g
+s/&#947;/γ/g
+s/&gamma;/γ/g
+s/&#948;/δ/g
+s/&delta;/δ/g
+s/&#949;/ε/g
+s/&epsilon;/ε/g
+s/&#950;/ζ/g
+s/&zeta;/ζ/g
+s/&#951;/η/g
+s/&eta;/η/g
+s/&#952;/θ/g
+s/&theta;/θ/g
+s/&#953;/ι/g
+s/&iota;/ι/g
+s/&#954;/κ/g
+s/&kappa;/κ/g
+s/&#955;/λ/g
+s/&lambda;/λ/g
+s/&#956;/μ/g
+s/&mu;/μ/g
+s/&#957;/ν/g
+s/&nu;/ν/g
+s/&#958;/ξ/g
+s/&xi;/ξ/g
+s/&#959;/ο/g
+s/&omicron;/ο/g
+s/&#960;/π/g
+s/&pi;/π/g
+s/&#961;/ρ/g
+s/&rho;/ρ/g
+s/&#962;/ς/g
+s/&sigmaf;/ς/g
+s/&#963;/σ/g
+s/&sigma;/σ/g
+s/&#964;/τ/g
+s/&tau;/τ/g
+s/&#965;/υ/g
+s/&upsilon;/υ/g
+s/&#966;/φ/g
+s/&phi;/φ/g
+s/&#967;/χ/g
+s/&chi;/χ/g
+s/&#968;/ψ/g
+s/&psi;/ψ/g
+s/&#969;/ω/g
+s/&omega;/ω/g
+s/&#977;/ϑ/g
+s/&thetasym;/ϑ/g
+s/&#978;/ϒ/g
+s/&upsih;/ϒ/g
+s/&#982;/ϖ/g
+s/&piv;/ϖ/g
+s/&#338;/Œ/g
+s/&OElig;/Œ/g
+s/&#339;/œ/g
+s/&oelig;/œ/g
+s/&#352;/Š/g
+s/&Scaron;/Š/g
+s/&#353;/š/g
+s/&scaron;/š/g
+s/&#376;/Ÿ/g
+s/&Yuml;/Ÿ/g
+s/&#402;/ƒ/g
+s/&fnof;/ƒ/g
+s/&#710;/ˆ/g
+s/&circ;/ˆ/g
+s/&#732;/˜/g
+s/&tilde;/˜/g
+s/&#8194;/ /g
+s/&ensp;/ /g
+s/&#8195;/ /g
+s/&emsp;/ /g
+s/&#8201;/ /g
+s/&thinsp;/ /g
+s/&#8204;/‌/g
+s/&zwnj;/‌/g
+s/&#8205;/‍/g
+s/&zwj;/‍/g
+s/&#8206;/‎/g
+s/&lrm;/‎/g
+s/&#8207;/‏/g
+s/&rlm;/‏/g
+s/&#8211;/–/g
+s/&ndash;/–/g
+s/&#8212;/—/g
+s/&mdash;/—/g
+s/&#8216;/‘/g
+s/&lsquo;/‘/g
+s/&#8217;/’/g
+s/&rsquo;/’/g
+s/&#8218;/‚/g
+s/&sbquo;/‚/g
+s/&#8220;/“/g
+s/&ldquo;/“/g
+s/&#8221;/”/g
+s/&rdquo;/”/g
+s/&#8222;/„/g
+s/&bdquo;/„/g
+s/&#8224;/†/g
+s/&dagger;/†/g
+s/&#8225;/‡/g
+s/&Dagger;/‡/g
+s/&#8226;/•/g
+s/&bull;/•/g
+s/&#8230;/…/g
+s/&hellip;/…/g
+s/&#8240;/‰/g
+s/&permil;/‰/g
+s/&#8242;/′/g
+s/&prime;/′/g
+s/&#8243;/″/g
+s/&Prime;/″/g
+s/&#8249;/‹/g
+s/&lsaquo;/‹/g
+s/&#8250;/›/g
+s/&rsaquo;/›/g
+s/&#8254;/‾/g
+s/&oline;/‾/g
+s/&#8364;/€/g
+s/&euro;/€/g
+s/&#8482;/™/g
+s/&#153;/™/g
+s/&trade;/™/g
+s/&#8592;/←/g
+s/&larr;/←/g
+s/&#8593;/↑/g
+s/&uarr;/↑/g
+s/&#8594;/→/g
+s/&rarr;/→/g
+s/&#8595;/↓/g
+s/&darr;/↓/g
+s/&#8596;/↔/g
+s/&harr;/↔/g
+s/&#8629;/↵/g
+s/&crarr;/↵/g
+s/&#8968;/⌈/g
+s/&lceil;/⌈/g
+s/&#8969;/⌉/g
+s/&rceil;/⌉/g
+s/&#8970;/⌊/g
+s/&lfloor;/⌊/g
+s/&#8971;/⌋/g
+s/&rfloor;/⌋/g
+s/&#9674;/◊/g
+s/&loz;/◊/g
+s/&#9824;/♠/g
+s/&spades;/♠/g
+s/&#9827;/♣/g
+s/&clubs;/♣/g
+s/&#9829;/♥/g
+s/&hearts;/♥/g
+s/&#9830;/♦/g
+s/&diams;/♦/g
+s/&#34;/"/g
+s/&quot;/"/g
+s/&#39;/'/g
+s/&apos;/'/g
+s/&#60;/</g
+s/&lt;/</g
+s/&#62;/>/g
+s/&gt;/>/g
+s/&#38;/&/g
+s/&amp;/&/g
+# http://www.w3schools.com/tags/ref_entities.asp
+# ^([^ \t]+)[ \t]+(&amp;[^;]*;)[ \t]+(&amp;[^;]*;).*$
+# s/\2/\1/g\ns/\3/\1/g