Skip to content

Commit 18f8b3e

Browse files
LordOfXenLordOfXen
authored
Add streaming support
1 parent a73d021 commit 18f8b3e

File tree

1 file changed

+236
-123
lines changed

1 file changed

+236
-123
lines changed

‎HashifyNETCLI/Program.cs‎

Lines changed: 236 additions & 123 deletions
Original file line numberDiff line numberDiff line change
@@ -922,11 +922,19 @@ public static int Main(string[] args)
922922
return 1;
923923
}
924924

925-
if (!(finalizedInput is byte[] inputArray))
926-
{
927-
Logger.Error("Input finalizer script must return a byte array.");
928-
return 1;
929-
}
925+
byte[] inputArray = (finalizedInput as byte[])!;
926+
Stream inputStream = (finalizedInput as Stream)!;
927+
if (inputArray == null && inputScript == null)
928+
{
929+
Logger.Error("Input finalizer script must either return a byte array or a stream.");
930+
return 1;
931+
}
932+
933+
if (inputStream != null && (!inputStream.CanRead || !inputStream.CanSeek))
934+
{
935+
Logger.Error("Input finalizer script must return a valid readable and seekable stream.");
936+
return 1;
937+
}
930938

931939
IReadOnlyList<FunctionVar> types = GetHashFunction(algorithm);
932940
if (types == null || types.Count < 1)
@@ -936,136 +944,241 @@ public static int Main(string[] args)
936944
return 1;
937945
}
938946

939-
foreach (FunctionVar fvar in types)
940-
{
941-
try
942-
{
943-
IHashConfigProfile? profile = null;
944-
if (configProfileQueries != null)
945-
{
946-
profile = GetConfigProfile(fvar, configProfileQueries);
947-
}
947+
try
948+
{
949+
foreach (FunctionVar fvar in types)
950+
{
951+
try
952+
{
953+
if (inputStream != null)
954+
{
955+
// Rewind the stream to the beginning for every algorithm to process the entire input.
956+
if (inputStream.Seek(0, SeekOrigin.Begin) != 0)
957+
{
958+
Logger.Error($"Failed to rewind the stream to the beginning for read operation by '{GetHashFunctionName(fvar)}'.");
959+
return 1;
960+
}
961+
}
948962

949-
IHashFunctionBase function;
963+
IHashConfigProfile? profile = null;
964+
if (configProfileQueries != null)
965+
{
966+
profile = GetConfigProfile(fvar, configProfileQueries);
967+
}
950968

951-
try
952-
{
953-
if (profile != null)
954-
{
955-
IHashConfigBase configProfile = profile.Create();
956-
if (configProfile == null)
957-
{
958-
Logger.Error($"Could not get the config profile instance for algorithm '{GetHashFunctionName(fvar)}'.");
959-
PrintAlgorithms();
960-
return 1;
961-
}
969+
IHashFunctionBase function;
962970

963-
function = HashFactory.Create(fvar.Function, configProfile);
964-
}
965-
else
966-
{
967-
JsonConfigProfile? jsonConfigProfile = null;
968-
if (_jsonConfigs != null && _jsonConfigs.Count > 0)
969-
{
970-
jsonConfigProfile = _jsonConfigs.Where(t => t.Type == fvar.Function).FirstOrDefault().Profiles?.Where(t => t.AsVar() == fvar).FirstOrDefault();
971+
try
972+
{
973+
if (profile != null)
974+
{
975+
IHashConfigBase configProfile = profile.Create();
976+
if (configProfile == null)
977+
{
978+
Logger.Error($"Could not get the config profile instance for algorithm '{GetHashFunctionName(fvar)}'.");
979+
PrintAlgorithms();
980+
return 1;
981+
}
982+
983+
function = HashFactory.Create(fvar.Function, configProfile);
984+
}
985+
else
986+
{
987+
JsonConfigProfile? jsonConfigProfile = null;
988+
if (_jsonConfigs != null && _jsonConfigs.Count > 0)
989+
{
990+
jsonConfigProfile = _jsonConfigs.Where(t => t.Type == fvar.Function).FirstOrDefault().Profiles?.Where(t => t.AsVar() == fvar).FirstOrDefault();
991+
992+
// Try again without the var name, in case there is a globalized config for this function type.
993+
if (jsonConfigProfile.HasValue && !jsonConfigProfile.Value.IsValid)
994+
{
995+
FunctionVar fvar2 = new FunctionVar(null, fvar.Function);
996+
jsonConfigProfile = _jsonConfigs.Where(t => t.Type == fvar.Function).FirstOrDefault().Profiles?.Where(t => t.AsVar() == fvar2).FirstOrDefault();
997+
}
998+
999+
if (jsonConfigProfile.HasValue && jsonConfigProfile.Value.Config == null && jsonConfigProfile.Value.Owner == null && jsonConfigProfile.Value.Name == null)
1000+
{
1001+
jsonConfigProfile = null;
1002+
}
1003+
}
1004+
1005+
if (jsonConfigProfile.HasValue)
1006+
{
1007+
function = HashFactory.Create(fvar.Function, jsonConfigProfile.Value.Config);
1008+
}
1009+
else
1010+
{
1011+
function = HashFactory.Create(fvar.Function);
1012+
}
1013+
}
1014+
}
1015+
catch (Exception ex)
1016+
{
1017+
Logger.Error("Could not create hash function instance: {0}", ex);
1018+
return 1;
1019+
}
9711020

972-
// Try again without the var name, in case there is a globalized config for this function type.
973-
if (jsonConfigProfile.HasValue && !jsonConfigProfile.Value.IsValid)
974-
{
975-
FunctionVar fvar2 = new FunctionVar(null, fvar.Function);
976-
jsonConfigProfile = _jsonConfigs.Where(t => t.Type == fvar.Function).FirstOrDefault().Profiles?.Where(t => t.AsVar() == fvar2).FirstOrDefault();
977-
}
1021+
if (function == null)
1022+
{
1023+
Logger.Error("Failed to create hash function instance.");
1024+
return 1;
1025+
}
9781026

979-
if (jsonConfigProfile.HasValue && jsonConfigProfile.Value.Config == null && jsonConfigProfile.Value.Owner == null && jsonConfigProfile.Value.Name == null)
980-
{
981-
jsonConfigProfile = null;
982-
}
983-
}
1027+
IHashValue result;
1028+
try
1029+
{
1030+
if (inputArray != null)
1031+
{
1032+
result = function.ComputeHash(inputArray);
1033+
}
1034+
else if (inputStream != null)
1035+
{
1036+
if (function is IStreamableHashFunctionBase streamableHash)
1037+
{
1038+
const int chunkSize = 67108864; // 64 megabytes
1039+
if (inputStream.Length < chunkSize)
1040+
{
1041+
var buffer = new byte[inputStream.Length];
1042+
inputStream.Read(buffer, 0, buffer.Length);
1043+
result = function.ComputeHash(buffer);
1044+
}
1045+
else
1046+
{
1047+
IBlockTransformer transformer = streamableHash.CreateBlockTransformer();
1048+
if (transformer == null)
1049+
{
1050+
Logger.Error($"Failed to create a valid block transformer for '{GetHashFunctionName(fvar)}'.");
1051+
return 1;
1052+
}
1053+
1054+
byte[] buffer = new byte[chunkSize];
1055+
int bytesRead;
1056+
long totalRead = 0;
1057+
double progress = 0;
1058+
1059+
const int progressBarWidth = 30;
1060+
1061+
int redZoneEnd = (int)(progressBarWidth * 0.33);
1062+
int yellowZoneEnd = (int)(progressBarWidth * 0.66);
1063+
1064+
while ((bytesRead = inputStream.Read(buffer, 0, buffer.Length)) > 0)
1065+
{
1066+
totalRead += bytesRead;
1067+
progress = ((double)totalRead / inputStream.Length) * 100.0d;
1068+
int filledBlocks = (int)Math.Round((progress / 100.0) * progressBarWidth);
1069+
1070+
int redBlocks = Math.Min(filledBlocks, redZoneEnd);
1071+
int yellowBlocks = Math.Max(0, Math.Min(filledBlocks, yellowZoneEnd) - redZoneEnd);
1072+
int greenBlocks = Math.Max(0, filledBlocks - yellowZoneEnd);
1073+
int emptyBlocks = progressBarWidth - filledBlocks;
1074+
1075+
Logger.LogDirect("\r", null, null);
1076+
1077+
Logger.LogDirect($"[{DateTimeOffset.UtcNow:MM/dd/yy-HH:mm:ss}] [{GetHashFunctionName(fvar)}] Transforming bytes: {totalRead}/{inputStream.Length} [", ConsoleColor.Gray, null);
1078+
1079+
if (redBlocks > 0) Logger.LogDirect(new string('█', redBlocks), ConsoleColor.Red, null);
1080+
if (yellowBlocks > 0) Logger.LogDirect(new string('█', yellowBlocks), ConsoleColor.DarkYellow, null);
1081+
if (greenBlocks > 0) Logger.LogDirect(new string('█', greenBlocks), ConsoleColor.DarkGreen, null);
1082+
if (emptyBlocks > 0) Logger.LogDirect(new string('-', emptyBlocks), ConsoleColor.DarkGray, null);
1083+
1084+
Logger.LogDirect($"] {$"{progress,3:F0}%"}", ConsoleColor.Gray, null);
1085+
1086+
transformer.TransformBytes(new ArraySegment<byte>(buffer, 0, bytesRead));
1087+
}
1088+
1089+
Logger.LogDirect("\n", null, null);
1090+
1091+
result = transformer.FinalizeHashValue();
1092+
}
1093+
}
1094+
else
1095+
{
1096+
if (inputStream.Length > int.MaxValue)
1097+
{
1098+
Logger.Error($"Unable to compute hash for '{GetHashFunctionName(fvar)}' with a stream size of over ~2 GB ({inputStream.Length} bytes). The all in once computation is designed for inputs smaller than 2 GB. Please pick algorithms that supports streaming data.");
1099+
return 1;
1100+
}
1101+
1102+
Logger.Warning($"Got a stream but the input algorithm '{GetHashFunctionName(fvar)}' does not support streaming computation. Trying to compute all data at once...");
1103+
1104+
byte[] buffer = new byte[inputStream.Length];
1105+
inputStream.Read(buffer, 0, buffer.Length);
1106+
1107+
result = function.ComputeHash(buffer);
1108+
}
1109+
}
1110+
else
1111+
{
1112+
result = null!;
1113+
}
1114+
}
1115+
catch (Exception ex)
1116+
{
1117+
Logger.Error("Hash computation failed: {0}", ex);
1118+
return 1;
1119+
}
9841120

985-
if (jsonConfigProfile.HasValue)
986-
{
987-
function = HashFactory.Create(fvar.Function, jsonConfigProfile.Value.Config);
988-
}
989-
else
990-
{
991-
function = HashFactory.Create(fvar.Function);
992-
}
993-
}
994-
}
995-
catch (Exception ex)
996-
{
997-
Logger.Error("Could not create hash function instance: {0}", ex);
998-
return 1;
999-
}
1121+
if (result == null)
1122+
{
1123+
Logger.Error($"Hash computation for '{GetHashFunctionName(fvar)}' returned a null result.");
1124+
return 1;
1125+
}
10001126

1001-
if (function == null)
1002-
{
1003-
Logger.Error("Failed to create hash function instance.");
1004-
return 1;
1005-
}
1127+
object finalizedOutput;
1128+
try
1129+
{
1130+
finalizedOutput = scriptEngine.FinalizeOutputScript(outputFinalizer, result);
1131+
}
1132+
catch (FailException ex)
1133+
{
1134+
Logger.Error(ex.Message);
1135+
return 2;
1136+
}
1137+
catch (Exception ex)
1138+
{
1139+
Logger.Error("Output finalizer script failed to execute: {0}", ex);
1140+
return 1;
1141+
}
10061142

1007-
IHashValue result;
1008-
try
1009-
{
1010-
result = function.ComputeHash(inputArray);
1011-
}
1012-
catch (Exception ex)
1013-
{
1014-
Logger.Error("Hash computation failed: {0}", ex);
1015-
return 1;
1016-
}
1143+
if (finalizedOutput == null)
1144+
{
1145+
Logger.Error("Output finalizer script returned null.");
1146+
return 1;
1147+
}
10171148

1018-
if (result == null)
1019-
{
1020-
Logger.Error($"Hash computation for '{GetHashFunctionName(fvar)}' returned a null result.");
1021-
return 1;
1149+
try
1150+
{
1151+
scriptEngine.OutputScript(outputScript, finalizedOutput, GetHashFunctionName(fvar));
1152+
}
1153+
catch (FailException ex)
1154+
{
1155+
Logger.Error(ex.Message);
1156+
return 2;
1157+
}
1158+
catch (Exception ex)
1159+
{
1160+
Logger.Error("Output script failed to execute: {0}", ex);
1161+
return 1;
1162+
}
10221163
}
1023-
1024-
object finalizedOutput;
1025-
try
1026-
{
1027-
finalizedOutput = scriptEngine.FinalizeOutputScript(outputFinalizer, result);
1028-
}
1029-
catch (FailException ex)
1030-
{
1031-
Logger.Error(ex.Message);
1032-
return 2;
1033-
}
1034-
catch (Exception ex)
1035-
{
1036-
Logger.Error("Output finalizer script failed to execute: {0}", ex);
1037-
return 1;
1038-
}
1039-
1040-
if (finalizedOutput == null)
1041-
{
1042-
Logger.Error("Output finalizer script returned null.");
1043-
return 1;
1164+
catch (Exception ex)
1165+
{
1166+
Logger.Error("An unexpected error occurred: {0}", ex);
1167+
return 1;
10441168
}
1169+
}
1170+
}
1171+
finally
1172+
{
1173+
if (inputStream != null)
1174+
{
1175+
inputStream.Close();
1176+
inputStream.Dispose();
1177+
inputStream = null!;
1178+
}
1179+
}
10451180

1046-
try
1047-
{
1048-
scriptEngine.OutputScript(outputScript, finalizedOutput, GetHashFunctionName(fvar));
1049-
}
1050-
catch (FailException ex)
1051-
{
1052-
Logger.Error(ex.Message);
1053-
return 2;
1054-
}
1055-
catch (Exception ex)
1056-
{
1057-
Logger.Error("Output script failed to execute: {0}", ex);
1058-
return 1;
1059-
}
1060-
}
1061-
catch (Exception ex)
1062-
{
1063-
Logger.Error("An unexpected error occurred: {0}", ex);
1064-
return 1;
1065-
}
1066-
}
1067-
1068-
return 0;
1181+
return 0;
10691182
}
10701183
}
10711184
}

0 commit comments

Comments
 (0)