fix a few compiler warnings
This commit is contained in:
181
Proc/HTMLToText.cs
Normal file
181
Proc/HTMLToText.cs
Normal file
@@ -0,0 +1,181 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using HtmlAgilityPack;
|
||||
|
||||
namespace WordpressEboobScraper2.Proc;
|
||||
|
||||
public static class HTMLToText
|
||||
{
|
||||
private static Regex REX_TAG1 = new Regex("<\\s*(link|style|script)[^>]*?/>", RegexOptions.Compiled);
|
||||
private static Regex REX_TAG2 = new Regex("<\\s*(link|style|script)[^>]*?>[^<>]*?<\\/\\s*\\1\\s*>", RegexOptions.Compiled);
|
||||
|
||||
private class PreceedingDomTextInfo
|
||||
{
|
||||
public PreceedingDomTextInfo(BoolWrapper isFirstTextOfDocWritten)
|
||||
{
|
||||
IsFirstTextOfDocWritten = isFirstTextOfDocWritten;
|
||||
}
|
||||
public bool WritePrecedingWhiteSpace { get; set; }
|
||||
public bool LastCharWasSpace { get; set; }
|
||||
public readonly BoolWrapper IsFirstTextOfDocWritten;
|
||||
public int ListIndex { get; set; }
|
||||
}
|
||||
|
||||
private class BoolWrapper
|
||||
{
|
||||
public BoolWrapper() { }
|
||||
public bool Value { get; set; }
|
||||
public static implicit operator bool(BoolWrapper boolWrapper)
|
||||
{
|
||||
return boolWrapper.Value;
|
||||
}
|
||||
public static implicit operator BoolWrapper(bool boolWrapper)
|
||||
{
|
||||
return new BoolWrapper { Value = boolWrapper };
|
||||
}
|
||||
}
|
||||
|
||||
public static string Convert(string path)
|
||||
{
|
||||
HtmlDocument doc = new HtmlDocument();
|
||||
doc.Load(path);
|
||||
return ConvertDoc(doc);
|
||||
}
|
||||
|
||||
public static string ConvertHtml(string html)
|
||||
{
|
||||
HtmlDocument doc = new HtmlDocument();
|
||||
html = REX_TAG1.Replace(html, " ");
|
||||
html = REX_TAG2.Replace(html, " ");
|
||||
doc.LoadHtml(html);
|
||||
return ConvertDoc(doc);
|
||||
}
|
||||
|
||||
public static string ConvertDoc(HtmlDocument doc)
|
||||
{
|
||||
using (StringWriter sw = new StringWriter())
|
||||
{
|
||||
ConvertTo(doc.DocumentNode, sw);
|
||||
sw.Flush();
|
||||
return sw.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
private static void ConvertContentTo(HtmlNode node, TextWriter outText, PreceedingDomTextInfo textInfo)
|
||||
{
|
||||
foreach (HtmlNode subnode in node.ChildNodes)
|
||||
{
|
||||
ConvertTo(subnode, outText, textInfo);
|
||||
}
|
||||
}
|
||||
|
||||
public static void ConvertTo(HtmlNode node, TextWriter outText)
|
||||
{
|
||||
ConvertTo(node, outText, new PreceedingDomTextInfo(false));
|
||||
}
|
||||
|
||||
private static void ConvertTo(HtmlNode node, TextWriter outText, PreceedingDomTextInfo textInfo)
|
||||
{
|
||||
string html;
|
||||
switch (node.NodeType)
|
||||
{
|
||||
case HtmlNodeType.Comment:
|
||||
// don't output comments
|
||||
break;
|
||||
case HtmlNodeType.Document:
|
||||
ConvertContentTo(node, outText, textInfo);
|
||||
break;
|
||||
case HtmlNodeType.Text:
|
||||
// script and style must not be output
|
||||
string parentName = node.ParentNode.Name;
|
||||
if ((parentName == "script") || (parentName == "style"))
|
||||
{
|
||||
break;
|
||||
}
|
||||
// get text
|
||||
html = ((HtmlTextNode)node).Text;
|
||||
// is it in fact a special closing node output as text?
|
||||
if (HtmlNode.IsOverlappedClosingElement(html)) break;
|
||||
|
||||
// check the text is meaningful and not a bunch of whitespaces
|
||||
if (html.Length == 0) break;
|
||||
|
||||
if (html.Trim().ToLower().StartsWith("<?xml") && html.Trim().ToLower().EndsWith("?>")) break;
|
||||
|
||||
if (!textInfo.WritePrecedingWhiteSpace || textInfo.LastCharWasSpace)
|
||||
{
|
||||
html = html.TrimStart();
|
||||
if (html.Length == 0) { break; }
|
||||
textInfo.IsFirstTextOfDocWritten.Value = textInfo.WritePrecedingWhiteSpace = true;
|
||||
}
|
||||
outText.Write(HtmlEntity.DeEntitize(Regex.Replace(html.TrimEnd(), @"\s{2,}", " ")));
|
||||
if (textInfo.LastCharWasSpace = char.IsWhiteSpace(html[html.Length - 1]))
|
||||
{
|
||||
outText.Write(' ');
|
||||
}
|
||||
break;
|
||||
case HtmlNodeType.Element:
|
||||
string endElementString = null;
|
||||
bool isInline;
|
||||
bool skip = false;
|
||||
int listIndex = 0;
|
||||
switch (node.Name)
|
||||
{
|
||||
case "nav":
|
||||
skip = true;
|
||||
isInline = false;
|
||||
break;
|
||||
case "body":
|
||||
case "section":
|
||||
case "article":
|
||||
case "aside":
|
||||
case "h1":
|
||||
case "h2":
|
||||
case "header":
|
||||
case "footer":
|
||||
case "address":
|
||||
case "main":
|
||||
case "div":
|
||||
case "span":
|
||||
case "p": // stylistic - adjust as you tend to use
|
||||
if (textInfo.IsFirstTextOfDocWritten) outText.Write("\r\n");
|
||||
endElementString = "\r\n";
|
||||
isInline = false;
|
||||
break;
|
||||
case "br":
|
||||
outText.Write("\r\n");
|
||||
skip = true;
|
||||
textInfo.WritePrecedingWhiteSpace = false;
|
||||
isInline = true;
|
||||
break;
|
||||
case "a":
|
||||
isInline = true;
|
||||
break;
|
||||
case "li":
|
||||
isInline = false;
|
||||
break;
|
||||
case "ol":
|
||||
listIndex = 1;
|
||||
goto case "ul";
|
||||
case "ul": //not handling nested lists any differently at this stage - that is getting close to rendering problems
|
||||
endElementString = "\r\n";
|
||||
isInline = false;
|
||||
break;
|
||||
case "img": //inline-block in reality
|
||||
isInline = true;
|
||||
break;
|
||||
default:
|
||||
isInline = true;
|
||||
break;
|
||||
}
|
||||
if (!skip && node.HasChildNodes)
|
||||
{
|
||||
ConvertContentTo(node, outText, isInline ? textInfo : new PreceedingDomTextInfo(textInfo.IsFirstTextOfDocWritten) { ListIndex = listIndex });
|
||||
}
|
||||
if (endElementString != null)
|
||||
{
|
||||
outText.Write(endElementString);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
59
Proc/ProcessHelper.cs
Normal file
59
Proc/ProcessHelper.cs
Normal file
@@ -0,0 +1,59 @@
|
||||
using System.Text;
|
||||
|
||||
namespace WordpressEboobScraper2.Proc;
|
||||
|
||||
public static class ProcessHelper
|
||||
{
|
||||
public static ProcessOutput ProcExecute(string command, string arguments, string workingDirectory = null)
|
||||
{
|
||||
var process = new System.Diagnostics.Process
|
||||
{
|
||||
StartInfo =
|
||||
{
|
||||
FileName = command,
|
||||
Arguments = arguments,
|
||||
WorkingDirectory = workingDirectory ?? string.Empty,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
ErrorDialog = false,
|
||||
}
|
||||
};
|
||||
|
||||
var builderOut = new StringBuilder();
|
||||
var builderErr = new StringBuilder();
|
||||
var builderBoth = new StringBuilder();
|
||||
|
||||
process.OutputDataReceived += (sender, args) =>
|
||||
{
|
||||
if (args.Data == null) return;
|
||||
|
||||
if (builderOut.Length == 0) builderOut.Append(args.Data);
|
||||
else builderOut.Append("\n" + args.Data);
|
||||
|
||||
if (builderBoth.Length == 0) builderBoth.Append(args.Data);
|
||||
else builderBoth.Append("\n" + args.Data);
|
||||
};
|
||||
|
||||
process.ErrorDataReceived += (sender, args) =>
|
||||
{
|
||||
if (args.Data == null) return;
|
||||
|
||||
if (builderErr.Length == 0) builderErr.Append(args.Data);
|
||||
else builderErr.Append("\n" + args.Data);
|
||||
|
||||
if (builderBoth.Length == 0) builderBoth.Append(args.Data);
|
||||
else builderBoth.Append("\n" + args.Data);
|
||||
};
|
||||
|
||||
process.Start();
|
||||
|
||||
process.BeginOutputReadLine();
|
||||
process.BeginErrorReadLine();
|
||||
|
||||
process.WaitForExit();
|
||||
|
||||
return new ProcessOutput($"{command} {arguments.Replace("\r", "\\r").Replace("\n", "\\n")}", process.ExitCode, builderOut.ToString(), builderErr.ToString(), builderBoth.ToString());
|
||||
}
|
||||
}
|
||||
21
Proc/ProcessOutput.cs
Normal file
21
Proc/ProcessOutput.cs
Normal file
@@ -0,0 +1,21 @@
|
||||
namespace WordpressEboobScraper2.Proc;
|
||||
|
||||
public struct ProcessOutput
|
||||
{
|
||||
public readonly string Command;
|
||||
public readonly int ExitCode;
|
||||
public readonly string StdOut;
|
||||
public readonly string StdErr;
|
||||
public readonly string StdCombined;
|
||||
|
||||
public ProcessOutput(string cmd, int ex, string stdout, string stderr, string stdcom)
|
||||
{
|
||||
Command = cmd;
|
||||
ExitCode = ex;
|
||||
StdOut = stdout;
|
||||
StdErr = stderr;
|
||||
StdCombined = stdcom;
|
||||
}
|
||||
|
||||
public override string ToString() => $"{Command}\n=> {ExitCode}\n\n[stdout]\n{StdOut}\n\n[stderr]\n{StdErr}";
|
||||
}
|
||||
Reference in New Issue
Block a user