admin/HawkeyeVision
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

check point

Browse Source
This commit is contained in:
meelstorm
2025-07-14 12:03:59 +02:00
commit d3cb790bd9
431 changed files with 44078 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
framework/Inspectron.Fastbuffer/Filesystems/AsyncFilesystem.cs Normal file
View File

@@ -0,0 +1,82 @@
using Inspectron.Fastbuffer.Interfaces;
using System.Collections.Concurrent;
namespace Inspectron.Fastbuffer.Filesystems;
public class AsyncFilesystem : IFilesystem
{
private readonly DirectFilesystem _directFilesystem = new();
private readonly ConcurrentDictionary<string, byte[]> _buffer = new();
private readonly BlockingCollection<(string path, byte[] data)> _writeQueue = new();
private readonly Thread _backgroundThread;
private readonly object _lock = new();
public AsyncFilesystem()
{
_backgroundThread = new Thread(ProcessWriteQueue) { IsBackground = true };
_backgroundThread.Name= "AsyncFilesystem";
_backgroundThread.Start();
}
public void Write(string path, byte[] data)
{
lock (_lock)
{
_buffer[path] = data;
_writeQueue.Add((path, data));
}
}
public byte[] Read(string path)
{
lock (_lock)
{
if (_buffer.TryGetValue(path, out var data))
{
return data;
}
}
return _directFilesystem.Read(path);
}
public void Delete(string path)
{
lock (_lock)
{
// pats is in buffer, so it's not written to disk yet
if (_buffer.TryRemove(path, out _))
{
return;
}
_directFilesystem.Delete(path);
}
}
private void ProcessWriteQueue()
{
foreach (var (path, data) in _writeQueue.GetConsumingEnumerable())
{
_directFilesystem.Write(path, data);
lock (_lock)
_buffer.TryRemove(path, out _);
}
}
public void Dispose()
{
_writeQueue.CompleteAdding();
_backgroundThread.Join();
}
public void Clear()
{
lock (_lock)
{
while (_writeQueue.TryTake(out _)) { }
}
}
}

31
framework/Inspectron.Fastbuffer/Filesystems/DirectFilesystem.cs Normal file
View File

@@ -0,0 +1,31 @@
using Inspectron.Fastbuffer.Interfaces;
using Serilog;
namespace Inspectron.Fastbuffer.Filesystems;
public class DirectFilesystem:IFilesystem
{
public void Write(string path, byte[] data)
{
// ensure path
Directory.CreateDirectory(Path.GetDirectoryName(path));
File.WriteAllBytes(path, data);
}
public byte[] Read(string path)
{
return File.ReadAllBytes(path);
}
public void Delete(string path)
{
Log.Debug($"Deleting {path}");
if (File.Exists(path))
File.Delete(path);
}
public void Dispose()
{
}
}

150
framework/Inspectron.Fastbuffer/IndexFile.cs Normal file
View File

@@ -0,0 +1,150 @@
using System.Text;
using Inspectron.Fastbuffer.Interfaces;
using Serilog;
namespace Inspectron.Fastbuffer;
public class IndexFile
{
private readonly string _indexFilePath;
private readonly int _bufferSize;
private readonly IFilesystem _filesystem;
private readonly string? _bufferPath;
private const int RecordSize = 512;
private const int IndexSize = 4;
private const int HeaderSize = 4;
public IndexFile(string path, int bufferSize, IFilesystem filesystem)
{
_indexFilePath = path;
_bufferSize = bufferSize;
_filesystem = filesystem;
ValidateFile();
_bufferPath = Path.Combine(Path.GetDirectoryName(_indexFilePath), "..");
}
private void ValidateFile()
{
if (!File.Exists(_indexFilePath))
{
CreateFile();
}
// check file size
var fileInfo = new FileInfo(_indexFilePath);
if (fileInfo.Length != _bufferSize * RecordSize + HeaderSize)
{
UpdateFile();
}
}
private void UpdateFile()
{
using var fileStream = File.Open(_indexFilePath, FileMode.Open);
var actualSize = fileStream.Length;
if (actualSize < _bufferSize+ HeaderSize)
{
fileStream.Seek(0, SeekOrigin.End);
var buffer = new byte[_bufferSize * RecordSize - actualSize];
fileStream.Write(buffer, HeaderSize, buffer.Length);
}
}
private void CreateFile()
{
using var fileStream = File.Create(_indexFilePath);
var buffer = new byte[_bufferSize* RecordSize + HeaderSize];
fileStream.Write(buffer, 0, buffer.Length);
fileStream.Seek(0, SeekOrigin.Begin);
var bytes = BitConverter.GetBytes(-1);
fileStream.Write(bytes, 0, bytes.Length);
}
public int GetCurrentArtifactId()
{
using var fileStream = File.Open(_indexFilePath, FileMode.Open);
using var reader = new BinaryReader(fileStream);
return reader.ReadInt32();
}
public void SetCurrentArtifactId(int id)
{
using var fileStream = File.Open(_indexFilePath, FileMode.Open);
using var writer = new BinaryWriter(fileStream);
writer.Write(id);
}
public string GetArtifactPath(int id)
{
using var fileStream = File.Open(_indexFilePath, FileMode.Open);
fileStream.Seek(id * RecordSize+HeaderSize, SeekOrigin.Begin);
byte[] bytes = new byte[RecordSize];
fileStream.Read(bytes, 0, RecordSize);
return Encoding.UTF8.GetString(bytes).TrimEnd('\0');
}
public int GetNextId()
{
var currentId = GetCurrentArtifactId();
if (currentId + 1 >= _bufferSize)
{
currentId = 0;
}
else
{
currentId++;
}
return currentId;
}
public void DeleteArtifact(int id)
{
var relPath = GetArtifactPath(id);
var filePath = Path.Combine(_bufferPath, relPath);
try
{
_filesystem.Delete(filePath);
}
catch (Exception e)
{
Log.Error("Error deleting artifact {id}: {message}",id,e.Message);
}
}
public void WriteNextArtifact(byte[] data, string fileName)
{
var id = GetNextId();
Log.Debug("Writing artifact {id}",id);
var existingPath = GetArtifactPath(id);
Log.Debug("Existing path: {existingPath}", existingPath);
DeleteArtifact(id);
var filePath = Path.Combine(_bufferPath, fileName);
_filesystem.Write(filePath, data);
SetCurrentArtifactId(id);
using var fileStream = File.Open(_indexFilePath, FileMode.Open);
fileStream.Seek(id * RecordSize+ HeaderSize, SeekOrigin.Begin);
var bytes = Encoding.UTF8.GetBytes(fileName);
fileStream.Write(bytes, 0, bytes.Length);
var padding = new byte[RecordSize - bytes.Length];
fileStream.Write(padding, 0, padding.Length);
}
public byte[] ReadArtifact(int id)
{
using var fileStream = File.Open(_indexFilePath, FileMode.Open);
fileStream.Seek(id * RecordSize+ HeaderSize, SeekOrigin.Begin);
var bytes = new byte[RecordSize];
fileStream.Read(bytes, 0, RecordSize);
var fileName = Encoding.UTF8.GetString(bytes).TrimEnd('\0');
var filePath = Path.Combine(_bufferPath, fileName);
return _filesystem.Read(filePath);
}
}

17
framework/Inspectron.Fastbuffer/Inspectron.Fastbuffer.csproj Normal file
View File

@@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<Folder Include="Interfaces\" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Serilog" Version="4.2.0" />
</ItemGroup>
</Project>

31
framework/Inspectron.Fastbuffer/Inspectron.Fastbuffer.sln Normal file
View File

@@ -0,0 +1,31 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.10.34928.147
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Inspectron.Fastbuffer", "Inspectron.Fastbuffer.csproj", "{697FB408-0899-423C-8D67-C4ABDBED0C09}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Inspectron.Fastbuffer.Tests", "..\Inspectron.Fastbuffer.Tests\Inspectron.Fastbuffer.Tests.csproj", "{5D1F5AAE-20DC-4C4A-B361-D1D42230BDF7}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{697FB408-0899-423C-8D67-C4ABDBED0C09}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{697FB408-0899-423C-8D67-C4ABDBED0C09}.Debug|Any CPU.Build.0 = Debug|Any CPU
{697FB408-0899-423C-8D67-C4ABDBED0C09}.Release|Any CPU.ActiveCfg = Release|Any CPU
{697FB408-0899-423C-8D67-C4ABDBED0C09}.Release|Any CPU.Build.0 = Release|Any CPU
{5D1F5AAE-20DC-4C4A-B361-D1D42230BDF7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5D1F5AAE-20DC-4C4A-B361-D1D42230BDF7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5D1F5AAE-20DC-4C4A-B361-D1D42230BDF7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5D1F5AAE-20DC-4C4A-B361-D1D42230BDF7}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {F3769CF4-894E-4BEE-9366-89E7505F27BF}
EndGlobalSection
EndGlobal

8
framework/Inspectron.Fastbuffer/Interfaces/IFilesystem.cs Normal file
View File

@@ -0,0 +1,8 @@
namespace Inspectron.Fastbuffer.Interfaces;
public interface IFilesystem:IDisposable
{
public void Write(string path, byte[] data);
public byte[] Read(string path);
public void Delete(string path);
}

38
framework/Inspectron.Fastbuffer/IsolatedRingbuffer.cs Normal file
View File

@@ -0,0 +1,38 @@
using Inspectron.Fastbuffer.Interfaces;
namespace Inspectron.Fastbuffer;
public class IsolatedRingbuffer: IDisposable
{
private readonly string _path;
private readonly int _bufferSize;
private readonly IFilesystem _filesystem;
private readonly string _indexPath;
private readonly IndexFile _index;
public const string BufferDirectory = ".rb";
public const string IndexFile = "index.idx";
public IsolatedRingbuffer(string path, int bufferSize, IFilesystem filesystem)
{
_path = path;
_bufferSize = bufferSize;
_filesystem = filesystem;
_indexPath = Path.Combine(_path, BufferDirectory,IndexFile );
Directory.CreateDirectory(Path.Combine(_path, BufferDirectory));
_index = new IndexFile(_indexPath, _bufferSize, _filesystem);
}
public void Write(byte[] data, string fileName)
{
_index.WriteNextArtifact(data, fileName);
}
public void Dispose()
{
_filesystem.Dispose();
}
}

52
framework/Inspectron.Fastbuffer/RingbufferRepository.cs Normal file
View File

@@ -0,0 +1,52 @@
using Inspectron.Fastbuffer.Interfaces;
namespace Inspectron.Fastbuffer
{
public class RingbufferRepository
{
private readonly string _path;
private readonly string _goodIndexPath;
private readonly string _badIndexPath;
private readonly IndexFile _goodIndex;
private readonly IndexFile _badIndex;
public const string BufferDirectory = ".rb";
public const string GoodIndexFile = "good.idx";
public const string BadIndexFile = "bad.idx";
public RingbufferRepository(string path, int goodBufferSize, int badBufferSize, IFilesystem filesystem)
{
_path = path;
_goodIndexPath = Path.Combine(_path, BufferDirectory, GoodIndexFile);
_badIndexPath = Path.Combine(_path, BufferDirectory, BadIndexFile);
Directory.CreateDirectory(Path.Combine(_path,BufferDirectory));
_goodIndex = new IndexFile(_goodIndexPath, goodBufferSize, filesystem);
_badIndex = new IndexFile(_badIndexPath, badBufferSize, filesystem);
var id= _goodIndex.GetCurrentArtifactId();
Console.WriteLine(@"RB size: "+id);
for (int i = 0; i < 3; i++)
{
var artifact = _goodIndex.GetArtifactPath(i);
Console.WriteLine(@"Artifact: "+artifact);
}
}
public IndexFile GoodIndex => _goodIndex;
public IndexFile BadIndex => _badIndex;
public void WriteGood(byte[] data, string fileName)
{
_goodIndex.WriteNextArtifact(data, fileName);
}
public void WriteBad(byte[] data, string fileName)
{
_badIndex.WriteNextArtifact(data, fileName);
}
}
}