First

include/fs/AbsPath.h

@@ -0,0 +1,53 @@
+#pragma once
+
+#include <tl/vector.h>
+#include "tl/string.h"
+#include "tl/path_system.h"
+#include "tl/rel_path.h"
+#include "tl/abs_path.h"
+
+namespace fs
+{
+
+// custom windows path system
+template<typename parse_separators, typename format_separator>
+struct WindowsSystemCustom : public tl::detail::path_system::base_path_system<parse_separators, format_separator>
+{
+	template<typename T>
+	static T format_absolute(const tl::vector<tl::string>& members);
+	static void parse_absolute(tl::vector<tl::string>& o_elements, const char* path, size_t size);
+	static bool validate_abs_path(const tl::vector<tl::string>& members);
+	static bool match(const char* path, size_t size);
+};
+TL_DECLARE_STRING_VECTOR(WindowsParseSeparators, "\\", "/");
+TL_DECLARE_STRING_LITERAL(WindowsFormatSeparator, "\\");
+using WindowsSystem = WindowsSystemCustom<WindowsParseSeparators, WindowsFormatSeparator>;
+
+// posix path system
+TL_DECLARE_STRING_LITERAL(PosixRootTag, "/");
+TL_DECLARE_STRING_VECTOR(PosixParseSeparators, "/");
+TL_DECLARE_STRING_LITERAL(PosixFormatSeparator, "/");
+using PosixSystem = tl::simple_path_system<PosixRootTag, PosixParseSeparators, PosixFormatSeparator>;
+
+// mac path system
+TL_DECLARE_STRING_LITERAL(UncRootTag, "\\\\");
+TL_DECLARE_STRING_VECTOR(UncParseSeparators, "\\", "/");
+TL_DECLARE_STRING_LITERAL(UncFormatSeparator, "\\");
+using UncSystem = tl::simple_path_system<UncRootTag, UncParseSeparators, UncFormatSeparator>;
+
+using AbsPath = tl::abs_path<tl::path_systems<WindowsSystem, PosixSystem, UncSystem> >;
+using RelPath = AbsPath::rel_path_type;
+
+}
+
+#include "fs/AbsPath.inl"
+
+template <>
+struct std::formatter<fs::RelPath>
+{
+	constexpr auto parse(format_parse_context& ctx) noexcept { return ctx.begin(); }
+	auto format(const fs::RelPath& p, std::format_context& ctx) const
+	{
+		return format_to(ctx.out(), "{}", p.get_as<fs::PosixSystem>());
+	}
+};

include/fs/AbsPath.inl

@@ -0,0 +1,51 @@
+#pragma once
+
+#include <locale>
+
+namespace fs
+{
+namespace detail
+{
+	TL_DECLARE_STRING_LITERAL(EmptyRootTag, "");
+}
+
+	// windows path system
+	template<typename parse_separators, typename format_separator>
+	template<typename T>
+	T WindowsSystemCustom<parse_separators, format_separator>::format_absolute(const tl::vector<tl::string>& members)
+	{
+		return tl::simple_path_system<detail::EmptyRootTag, parse_separators, format_separator>::template format_absolute<T>(members);
+	}
+
+	template<typename parse_separators, typename format_separator>
+	void WindowsSystemCustom<parse_separators, format_separator>::parse_absolute(tl::vector<tl::string>& o_elements, const char* path, size_t size)
+	{
+		// On windows, the drive letter case is undefined. To avoid issues (hashing, etc), we always convert the drive letter to uppercase
+		tl::simple_path_system<detail::EmptyRootTag, parse_separators, format_separator>::parse_absolute(o_elements, path, size);
+		if (!o_elements.empty())
+		{
+			tl::string& str = o_elements.front();
+			if (str.length() == 2 && str[1] == ':')
+			{
+				char strUp[2];
+				strUp[0] = tl::ascii::toupper(str[0]);
+				strUp[1] = ':';
+				str = tl::string(strUp, 2);
+			}
+		}
+	}
+
+	template<typename parse_separators, typename format_separator>
+	bool WindowsSystemCustom<parse_separators, format_separator>::match(const char* path, size_t size)
+	{
+        return size >= 2 && std::isalpha(path[0], std::locale()) && path[1] == ':';
+	}
+
+	template<typename parse_separators, typename format_separator>
+	bool WindowsSystemCustom<parse_separators, format_separator>::validate_abs_path(const tl::vector<tl::string>& members)
+	{
+		return !members.empty() && match(members[0].c_str(), members[0].size());
+	}
+
+//////////////////////////////////////////////////////////////////////////
+}

include/fs/Api.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include "tl/platform.h"
+
+#if defined FS_BUILD_SHARED_LIB
+
+	#if defined TL_TOOLCHAIN_MSC
+	#	define FS_API __declspec(dllexport)
+	#else
+	# 	define FS_API
+	#endif
+
+#elif defined FS_USE_SHARED_LIB
+
+	#if defined TL_TOOLCHAIN_MSC
+	#	define FS_API __declspec(dllimport)
+	#else
+	#	define FS_API
+	#endif //
+
+#else
+#	define FS_API
+#endif

include/fs/BufferedFileSource.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "fs/BufferedSource.h"
+#include "fs/FileSource.h"
+
+namespace fs
+{
+/**
+ * Buffered version of the FileSource ideal for doing random access to file.
+ * for further information please take a look to BufferedSource.
+ */
+using BufferedFileSource = BufferedSource<FileSource>;
+
+}

include/fs/BufferedSource.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include "fs/ISource.h"
+#include "tl/memory_buffer.h"
+
+namespace fs
+{
+/**
+ * ISource interface implementation where the access to the underneath source
+ * is done in buffered mode. This class must be used when there a lot of tiny
+ * random access to the source. For further information please check ISource.
+ */
+template<typename SourceType>
+class BufferedSource final : public ISource
+{
+public:
+	explicit BufferedSource(SourceType source, size_t bufferSize = 4096u);
+
+	BufferedSource(BufferedSource&& other) = default;
+	BufferedSource& operator=(BufferedSource&& other) = default;
+	BufferedSource(const BufferedSource& other) = delete;
+	BufferedSource& operator=(const BufferedSource& other) = delete;
+
+	tl::optional<Error> getLastError() const override;
+
+	size_t read(tl::span<uint8_t> data) override;
+
+	void seekBeg(uint64_t offset) override;
+	void seekRel(int64_t offset) override;
+
+	uint64_t tell() const override;
+
+	uint64_t getSize() const override;
+
+	bool isEOS() const override;
+
+	size_t getPreferredBufferSize() const override;
+private:
+	SourceType m_source;
+	uint64_t m_bufferStartOffset = 0;
+	uint64_t m_bufferOffset = 0;
+	bool m_endOfSource = false;
+	tl::memory_buffer m_buffer;
+	mutable tl::optional<Error> m_optLastError;
+};
+
+}

include/fs/BufferedSource.inl

@@ -0,0 +1,150 @@
+
+#include "fs/BufferedSource.h"
+#include <algorithm>
+
+namespace fs
+{
+//////////////////////////////////////////////////////////////////////////
+
+template< typename SourceType>
+BufferedSource<SourceType>::BufferedSource(SourceType source, size_t bufferSize)
+	: m_source(std::move(source))
+{
+	if (bufferSize < 8)
+		bufferSize = 8;
+
+	m_buffer.reserve(bufferSize);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+template< typename SourceType>
+tl::optional<Error> BufferedSource<SourceType>::getLastError() const
+{
+	tl::optional<Error> error = m_optLastError;
+	m_optLastError = tl::nullopt;
+	return error;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+template< typename SourceType>
+size_t BufferedSource<SourceType>::read(tl::span<uint8_t> data)
+{
+	if (data.empty() || m_endOfSource)
+		return 0;
+
+	auto bufferPtr = data.data();
+	size_t totalSize = 0;
+	size_t pendingSize = data.size();
+
+	while (pendingSize)
+	{
+		if (m_buffer.empty() || m_bufferOffset >= m_buffer.size())
+		{
+			m_buffer.resize_uninitialized(m_buffer.capacity());
+			m_bufferStartOffset = m_source.tell();
+			m_bufferOffset = 0;
+			const size_t readSize = m_source.read(m_buffer);
+			m_buffer.resize_uninitialized(readSize);
+			if (readSize == 0)
+			{
+				m_optLastError = m_source.getLastError();
+				m_endOfSource = !m_optLastError.has_value(); //no error? then EOS
+				return totalSize;
+			}
+		}
+
+		TL_ASSERT(m_buffer.size());
+		TL_ASSERT(m_buffer.size() > m_bufferOffset);
+		const size_t frameSize = tl::min(m_buffer.size() - m_bufferOffset, pendingSize);
+		TL_ASSERT(frameSize);
+
+		memcpy(bufferPtr, m_buffer.data() + m_bufferOffset, frameSize);
+		bufferPtr += frameSize;
+
+		TL_ASSERT(data.size() >= frameSize);
+		pendingSize -= frameSize;
+
+		totalSize += frameSize;
+		TL_ASSERT(totalSize <= data.size());
+
+		m_bufferOffset += frameSize;
+	}
+
+	m_endOfSource = tell() >= getSize();
+
+	return totalSize;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+template< typename SourceType>
+void BufferedSource<SourceType>::seekBeg(uint64_t offset)
+{
+	offset = tl::min(offset, m_source.getSize());
+	if (offset < m_bufferStartOffset || offset >= m_bufferStartOffset + m_buffer.size())
+	{
+		m_source.seekBeg(offset);
+		m_optLastError = m_source.getLastError();
+		m_buffer.clear();
+		m_bufferOffset = 0;
+		m_bufferStartOffset = m_source.tell();
+	}
+	else
+		m_bufferOffset = offset - m_bufferStartOffset;
+
+	m_endOfSource = tell() >= getSize();
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+template< typename SourceType>
+void BufferedSource<SourceType>::seekRel(int64_t offset)
+{
+	if (!offset)
+		return;
+
+	if (offset != 0)
+	{
+		int64_t newOffset = tell();
+		newOffset += offset;
+		newOffset = tl::max<int64_t>(newOffset, 0);
+		seekBeg(static_cast<uint64_t>(tl::min<int64_t>(newOffset, getSize())));
+	}
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+template< typename SourceType>
+uint64_t BufferedSource<SourceType>::tell() const
+{
+	return m_bufferStartOffset + m_bufferOffset;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+template< typename SourceType>
+uint64_t BufferedSource<SourceType>::getSize() const
+{
+	return m_source.getSize();
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+template< typename SourceType>
+bool BufferedSource<SourceType>::isEOS() const
+{
+	return m_endOfSource;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+template< typename SourceType>
+size_t BufferedSource<SourceType>::getPreferredBufferSize() const
+{
+	return m_buffer.size();
+}
+
+}
+

include/fs/CRCStream.h

@@ -0,0 +1,25 @@
+#pragma once
+#include "Error.h"
+#include "fs/Api.h"
+#include "fs/AbsPath.h"
+
+namespace fs
+{
+//////////////////////////////////////////////////////////////////////////
+
+class IStreamSource;
+class IFilesystem;
+
+FS_API uint32_t computeCRC32(uint32_t crc, tl::span<const uint8_t> data);
+FS_API uint64_t computeCRC64(uint64_t crc, tl::span<const uint8_t> data);
+
+typedef tl::result<uint32_t, Error> CRC32StreamResult;
+FS_API CRC32StreamResult crc32Stream(IStreamSource& source, uint32_t crc = 0, size_t bufferSize = 0);
+FS_API CRC32StreamResult crc32File(const IFilesystem& filesystem, const AbsPath& filePath, uint32_t crc = 0, size_t bufferSize = 0);
+
+typedef tl::result<uint64_t, Error> CRC64StreamResult;
+FS_API CRC64StreamResult crc64Stream(IStreamSource& source, uint64_t crc = 0, size_t bufferSize = 0);
+FS_API CRC64StreamResult crc64File(const IFilesystem& filesystem, const AbsPath& filePath, uint64_t crc = 0, size_t bufferSize = 0);
+
+//////////////////////////////////////////////////////////////////////////
+}

include/fs/CopyStream.h

@@ -0,0 +1,24 @@
+#pragma once
+#include "Error.h"
+#include "fs/Api.h"
+#include "fs/ProcessStream.h"
+#include "tl/result.h"
+#include "fs/AbsPath.h"
+
+namespace fs
+{
+//////////////////////////////////////////////////////////////////////////
+
+class IStreamSink;
+class IStreamSource;
+class IFilesystem;
+
+using CopyStreamResult = tl::result<uint64_t, Error>;
+FS_API CopyStreamResult copyStream(IStreamSink& sink, IStreamSource& source, size_t bufferSize = 0);
+FS_API CopyStreamResult copyStream(IStreamSink& sink, IStreamSource& source, const ProcessDataCallback& dataCallback, size_t bufferSize = 0);
+FS_API CopyStreamResult copyFile(IStreamSink& sink, const IFilesystem& filesystem, const AbsPath& filePath, size_t bufferSize = 0);
+FS_API CopyStreamResult copyFile(IStreamSink& sink, const IFilesystem& filesystem, const AbsPath& filePath, const ProcessDataCallback& dataCallback, size_t bufferSize = 0);
+FS_API CopyStreamResult copyFile(IFilesystem& dstFilesystem, const AbsPath& dstFilePath, const IFilesystem& srcFilesystem, const AbsPath& srcFilePath, size_t bufferSize = 0);
+
+//////////////////////////////////////////////////////////////////////////
+}

include/fs/CustomFilesystem.h

@@ -0,0 +1,99 @@
+#pragma once
+
+#include "fs/IPack.h"
+#include "fs/IFilesystem.h"
+#include "fs/Error.h"
+#include "fs/Mode.h"
+#include "fs/Api.h"
+#include "tl/vector.h"
+#include "tl/identifier.h"
+
+namespace fs
+{
+
+class FS_API CustomFilesystem final : public IFilesystem
+{
+public:
+	CustomFilesystem() = default;
+	CustomFilesystem(CustomFilesystem&&) = delete;
+	CustomFilesystem& operator=(CustomFilesystem&&) = delete;
+	CustomFilesystem(const CustomFilesystem&) = delete;
+	CustomFilesystem& operator=(const CustomFilesystem&) = delete;
+
+	//////////////////////////////////////////////////////////////////////////
+	// New API
+
+	TL_DECLARE_INTEGRAL_ID(PackId, uint64_t, 0)
+
+	tl::result<PackId> mountFront(AbsPath mountPoint, tl::unique_ref<IPack> pack);
+	tl::result<PackId> mountBack(AbsPath mountPoint, tl::unique_ref<IPack> pack);
+
+	//unmounts all the packs for this point. It basically deletes the mount point
+	cppcoro::generator<tl::unique_ref<IPack>> unmountAll();
+
+	//When unmounting, the Filesystem returns the pack back to you (if found)
+	tl::unique_ptr<IPack> unmount(PackId packId);
+
+	//Returns the physical location of the virtual pack. 
+	ConvertToNativePathResult convertToNativePath(const AbsPath& path) const override;
+
+	OpenSourceResult openSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override;
+	OpenStreamSourceResult openStreamSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override;
+	OpenMapSourceResult openMapSource(const AbsPath& path, MapView mapView = MapView(), SourceFlags flags = SourceFlags()) const override;
+	OpenSinkResult openSink(const AbsPath& path, Mode mode, SinkFlags flags = SinkFlags()) override;
+	OpenStreamSinkResult openStreamSink(const AbsPath& path, Mode mode, SinkFlags flags = SinkFlags()) override;
+	OpenMapSinkResult openMapSink(const AbsPath& path, Mode mode, size_t size, SinkFlags flags = SinkFlags()) override;
+
+	IsFileResult isFile(const AbsPath& path) const override;
+	IsFolderResult isFolder(const AbsPath& path) const override;
+	ExistsResult exists(const AbsPath& path) const override;
+
+	MakeFolderResult makeFolder(const AbsPath& path) override;
+
+	RenameResult rename(const AbsPath& path, const AbsPath& newPath) override;
+
+	RemoveResult remove(const AbsPath& path) override;
+	RemoveRecursivelyResult removeRecursively(const AbsPath& path) override;
+
+	CopyResult copy(const AbsPath& path, const AbsPath& newPath) override;
+
+	MakeHardLinkResult makeHardLink(const AbsPath& sourcePath, const AbsPath& linkPath) override;
+	MakeSoftLinkResult makeSymLink(const AbsPath& sourcePath, const AbsPath& linkPath) override;
+
+	SetWriteTimeResult setWriteTime(const AbsPath& path, time_t time) override;
+
+	GetStatResult getStat(const AbsPath& path) const override;
+
+	cppcoro::generator<EnumerateEntry> enumerate(const AbsPath& path) const override;
+	cppcoro::generator<EnumerateEntry> enumerateRecursively(const AbsPath& path) const override;
+
+private:
+	struct PackData
+	{
+		PackId id;
+		tl::unique_ref<IPack> pack;
+		AbsPath mountPoint;
+
+		explicit PackData(tl::unique_ref<IPack>&& pack) noexcept
+			: pack(std::move(pack))
+		{}
+
+		PackData(PackData&& other) noexcept = default;
+		PackData& operator=(PackData&& other) noexcept = default;
+		PackData(const PackData& other) noexcept = delete;
+		PackData& operator=(const PackData& other) noexcept = delete;
+	};
+	tl::vector<PackData> m_packsData;
+
+	void convertToPackPath(AbsPath& packPath, const AbsPath& path, const AbsPath& mountPoint) const;
+
+	enum class MountPolicy
+	{
+		Front, 		//the pack will be mounted over packs from the same mount point. This pack will take precedence
+		Back 		//the pack will be at the bottom of existing packs from the same mount point. Its streams will be searched last
+	};
+	tl::result<PackId> mount(AbsPath mountPoint, tl::unique_ref<IPack> pack, MountPolicy policy);
+
+};
+
+}

include/fs/DeflateStreamSink.h

@@ -0,0 +1,54 @@
+#pragma once
+#include "fs/IStreamSink.h"
+#include "tl/memory_buffer.h"
+#include "fs/Api.h"
+#include "tl/ptr.h"
+
+namespace fs
+{
+//////////////////////////////////////////////////////////////////////////
+
+class FS_API DeflateStreamSink final: public IStreamSink
+{
+public:
+	explicit DeflateStreamSink(IStreamSink& sink); //non owning
+	explicit DeflateStreamSink(tl::unique_ref<IStreamSink> sink); //owning
+	~DeflateStreamSink() override;
+
+	//compression level - 0 to 9
+	void setCompressionLevel(uint8_t level);
+
+	tl::optional<Error> getLastError() const override;
+	size_t write(tl::span<const uint8_t> data) override;
+
+	size_t getPreferredBufferSize() const override;
+
+	uint64_t getSize() const override;
+
+	//Finish is also called by the destructor
+	void finish();
+	tl::unique_ref<IStreamSink> finishAndExtract();
+
+private:
+	tl::unique_ptr<IStreamSink> m_ownedSink;
+	IStreamSink& m_sink;
+
+	uint8_t m_level = 6;
+	mutable tl::optional<Error> m_optLastError;
+
+	void initializeDeflate();
+	bool flush(bool finalize);
+
+	struct ZLib;
+	tl::unique_ptr<ZLib> m_zlib;
+
+	tl::memory_buffer m_buffer;
+	size_t m_bufferOffset = 0;
+	uint64_t m_size = 0;
+
+	tl::memory_buffer m_outBuffer;
+	bool m_isFinished = false;
+};
+
+//////////////////////////////////////////////////////////////////////////
+}

include/fs/DeflateStreamSource.h

@@ -0,0 +1,41 @@
+#pragma once
+#include "fs/IStreamSource.h"
+#include "tl/memory_buffer.h"
+#include "fs/Api.h"
+#include "tl/ptr.h"
+
+namespace fs
+{
+
+class FS_API DeflateStreamSource final : public IStreamSource
+{
+public:
+	explicit DeflateStreamSource(IStreamSource& source); //non-owning
+	explicit DeflateStreamSource(tl::unique_ref<IStreamSource> source); //owning
+	~DeflateStreamSource() override;
+
+	tl::optional<Error> getLastError() const override;
+	size_t read(tl::span<uint8_t> data) override;
+
+	bool isEOS() const override;
+
+	size_t getPreferredBufferSize() const override;
+
+	tl::unique_ref<IStreamSource> extract();
+
+private:
+	tl::unique_ptr<IStreamSource> m_ownedSource;
+	IStreamSource& m_source;
+
+	mutable tl::optional<Error> m_optLastError;
+
+	void initializeDeflate();
+
+	struct ZLib;
+	tl::unique_ptr<ZLib> m_zlib;
+
+	tl::memory_buffer m_buffer;
+};
+
+
+}

include/fs/Error.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include "tl/result.h"
+
+namespace fs
+{
+
+class ISource;
+class IStreamSource;
+class IMapSource;
+class ISink;
+class IStreamSink;
+class IMapSink;
+
+enum class ErrorCode : uint8_t
+{
+	BadPath,
+	InvalidArgument,
+	NotFound,
+	NotAllowed,
+	SystemError,
+	NotSupported
+};
+typedef tl::error<ErrorCode> Error;
+
+}
+

include/fs/FS.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <tl/vector.h>
+#include <cstring>//c78 - fix for error: 'strlen' is not a member of 'std' on android
+
+#include "tl/platform.h"
+#include "tl/string.h"
+#include "tl/ptr.h"
+#include "tl/result.h"
+
+//#define FS_H_INCLUDED_CORRECTLY
+
+//to disable the MSVC warning about inheriting via dominance
+#if defined(_MSC_VER)
+#pragma warning(disable:4250)
+#endif
+
+#include "fs/Api.h"
+#include "Mode.h"
+#include "Error.h"
+
+#include "native/NativeUtils.h"
+#include "AbsPath.h"
+
+
+#include "ISource.h"
+#include "ISink.h"
+
+#include "IPack.h"
+#include "IWritablePack.h"
+#include "IFilesystem.h"
+
+//native implementation
+//#include "native/FolderFilesystem.h"
+//#include "native/FolderPack.h"
+
+#include "native/FileSource.h"
+#include "native/FileSink.h"
+#include "native/FileMapSource.h"
+#include "native/FileMapSink.h"
+//
+#include "MemorySource.h"
+#include "MemorySink.h"
+
+//clean up the preprocessor mess
+#undef FS_H_INCLUDED_CORRECTLY

include/fs/FileMapSink.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include "tl/ptr.h"
+#include "fs/IMapSink.h"
+#include "fs/AbsPath.h"
+#include "tl/flag_set2.h"
+#include "fs/Mode.h"
+#include "fs/Api.h"
+
+namespace fs
+{
+
+class NativeFilesystem;
+
+class FS_API FileMapSink final : public IMapSink
+{
+	friend class NativeFilesystem;
+public:
+	FileMapSink(const AbsPath& filepath, size_t size, Mode mode = Mode::CreateOrOpenAndClear, Flags flags = Flags());
+	~FileMapSink() override;
+
+	bool isValid() const;
+	tl::optional<Error> getLastError() const override;
+
+	size_t write(tl::span<const uint8_t> data) override;
+	void seekBeg(uint64_t offset) override;
+	void seekRel(int64_t offset) override;
+	uint64_t tell() const override;
+	uint64_t getSize() const override;
+	tl::span<uint8_t> map(size_t size) override;
+	size_t getPreferredBufferSize() const override;
+
+private:
+	uint64_t	m_offset = 0;
+	mutable tl::optional<Error> m_optLastError;
+
+	struct Impl;
+	tl::unique_ptr<Impl> m_impl;
+};
+
+}
+

include/fs/FileMapSource.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include "tl/ptr.h"
+
+#include "fs/IMapSource.h"
+#include "fs/AbsPath.h"
+#include "fs/Api.h"
+
+namespace fs
+{
+
+class NativeFilesystem;
+
+class FS_API FileMapSource final : public IMapSource
+{
+	friend class NativeFilesystem;
+public:
+	explicit FileMapSource(const AbsPath& filepath);
+	FileMapSource(const AbsPath& filepath, uint64_t start, size_t size);
+
+	~FileMapSource() override;
+
+	bool isValid() const;
+	tl::optional<Error> getLastError() const override;
+
+	size_t read(tl::span<uint8_t> data) override;
+
+	void seekBeg(uint64_t offset) override;
+	void seekRel(int64_t offset) override;
+
+	uint64_t tell() const override;
+	uint64_t getSize() const override;
+
+	tl::span<const uint8_t> map(size_t size) override;
+
+	bool isEOS() const override;
+
+	size_t getPreferredBufferSize() const override;
+
+private:
+	uint64_t m_offset = 0;
+	mutable tl::optional<Error> m_optLastError;
+
+	struct Impl;
+	tl::unique_ptr<Impl> m_impl;
+};
+
+}
+

include/fs/FileSink.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include "fs/ISink.h"
+#include "fs/Mode.h"
+#include "fs/AbsPath.h"
+#include "tl/flag_set2.h"
+#include "fs/Api.h"
+
+namespace fs
+{
+
+class FS_API FileSink final : public ISink
+{
+public:
+	FileSink(const AbsPath& filepath, Mode mode, Flags flags = Flags()) noexcept;
+	~FileSink() noexcept override;
+
+	FileSink(FileSink&& other) noexcept;
+	FileSink& operator=(FileSink&& other) noexcept;
+
+	bool isValid() const noexcept;
+	tl::optional<Error> getLastError() const override;
+
+	size_t write(tl::span<const uint8_t> data) override;
+
+	void seekBeg(uint64_t offset) override;
+	void seekRel(int64_t offset) override;
+
+	uint64_t tell() const override;
+	uint64_t getSize() const override;
+
+	size_t getPreferredBufferSize() const override;
+
+	bool resize(uint64_t newSize) noexcept;
+
+private:
+	//static tl::result<Handle, Error> openFile(const AbsPath& filepath, Mode mode, Flags flags) noexcept;
+
+	void close() noexcept;
+
+	bool m_isConstructed = false;
+	std::array<uint64_t, 4> m_arena;
+
+	uint64_t m_offset = 0;
+	mutable tl::optional<Error> m_optLastError;
+};
+}

include/fs/FileSource.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include "fs/ISource.h"
+#include "fs/AbsPath.h"
+#include "fs/Api.h"
+
+namespace fs
+{
+
+class FS_API FileSource final : public ISource
+{
+public:
+	explicit FileSource(const AbsPath& i_filepath) noexcept;
+	FileSource(const AbsPath& i_filepath, Flags i_flags) noexcept;
+	~FileSource() noexcept override;
+
+	FileSource(FileSource&& i_other) noexcept;
+	FileSource& operator=(FileSource&& i_other) noexcept;
+
+	bool isValid() const noexcept;
+	tl::optional<Error> getLastError() const override;
+
+	size_t read(tl::span<uint8_t> data) override;
+
+	void seekBeg(uint64_t i_offset) override;
+	void seekRel(int64_t i_offset) override;
+
+	uint64_t tell() const override;
+	uint64_t getSize() const override;
+
+	bool isEOS() const override;
+
+	size_t getPreferredBufferSize() const override;
+
+private:
+	FileSource() noexcept = default;
+	void swap(FileSource& i_other) noexcept;
+
+	void close() noexcept;
+
+	bool m_isConstructed = false;
+	std::array<uint64_t, 4> m_arena;
+
+	uint64_t m_offset = 0;
+	mutable uint64_t m_size = uint64_t(-1);
+	mutable tl::optional<Error> m_optLastError;
+};
+}
+

include/fs/FolderPack.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include "fs/IPack.h"
+#include "fs/Api.h"
+
+namespace fs
+{
+//////////////////////////////////////////////////////////////////////////
+
+class FS_API FolderPack : virtual public IPack
+{
+public:
+	explicit FolderPack(AbsPath location);
+	FolderPack(tl::lent_ref<IFilesystem> filesystem, AbsPath location);
+
+	~FolderPack() override = default;
+
+	OpenSourceResult openSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override;
+	OpenStreamSourceResult openStreamSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override;
+	OpenMapSourceResult openMapSource(const AbsPath& path, MapView mapView = MapView(), SourceFlags flags = SourceFlags()) const override;
+
+	IsFileResult isFile(const AbsPath& path) const override;
+	IsFolderResult isFolder(const AbsPath& path) const override;
+	ExistsResult exists(const AbsPath& path) const override;
+
+	GetStatResult getStat(const AbsPath& path) const override;
+
+	cppcoro::generator<EnumerateEntry> enumerate(const AbsPath& path) const override;
+	cppcoro::generator<EnumerateEntry> enumerateRecursively(const AbsPath& path) const override;
+
+	ConvertToNativePathResult convertToNativePath(const AbsPath& path) const override;
+
+protected:
+	AbsPath convertToUnderlyingPath(const AbsPath& path) const;
+	tl::lent_ref<IFilesystem> m_filesystem;
+	AbsPath m_location;
+};
+
+//////////////////////////////////////////////////////////////////////////
+}

include/fs/IFilesystem.h

@@ -0,0 +1,126 @@
+#pragma once
+
+#include "fs/AbsPath.h"
+#include "tl/ptr.h"
+
+#include "fs/Error.h"
+#include "fs/Mode.h"
+
+#include "fs/ISource.h"
+#include "fs/IMapSource.h"
+#include "fs/ISink.h"
+#include "fs/IMapSink.h"
+#include "fs/Api.h"
+#include <cppcoro/generator.hpp>
+
+
+namespace fs
+{
+using OpenSourceResult = tl::result<tl::unique_ref<ISource>, Error>;
+using OpenStreamSourceResult = tl::result<tl::unique_ref<IStreamSource>, Error>;
+using OpenMapSourceResult = tl::result<tl::unique_ref<IMapSource>, Error>;
+using OpenSinkResult = tl::result<tl::unique_ref<ISink>, Error>;
+using OpenStreamSinkResult = tl::result<tl::unique_ref<IStreamSink>, Error>;
+using OpenMapSinkResult = tl::result<tl::unique_ref<IMapSink>, Error>;
+using ConvertToNativePathResult = tl::result<AbsPath, Error>;
+using IsFileResult = tl::result<bool, Error>;
+using IsFolderResult = tl::result<bool, Error>;
+using ExistsResult = tl::result<bool, Error>;
+using MakeFolderResult = tl::result<void, Error>;
+using RenameResult = tl::result<void, Error>;
+using RemoveResult = tl::result<void, Error>;
+using CopyResult = tl::result<void, Error>;
+using MakeHardLinkResult = tl::result<void, Error>;
+using MakeSoftLinkResult = tl::result<void, Error>;
+using RemoveRecursivelyResult = tl::result<void, Error>;
+struct Stat
+{
+	uint64_t size = 0;
+	time_t mTime = 0;
+	time_t aTime = 0;
+};
+using GetStatResult = tl::result<Stat, Error>;
+using SetWriteTimeResult = tl::result<void, Error>;
+
+struct EnumerateEntry
+{
+	RelPath path;
+	bool isFolder = false;
+};
+
+class FS_API IFilesystem
+{
+public:
+	//////////////////////////////////////////////////////////////////////////
+
+	virtual ~IFilesystem() = default;
+
+	using SourceFlag = ISource::Flag;
+	using SourceFlags = ISource::Flags;
+
+	struct MapView
+	{
+		MapView(uint64_t offset = 0, size_t size = 0)
+			: offset(offset), size(size) {}
+		uint64_t offset;
+		size_t size; //zero size means full size
+	};
+
+	//Opens a Read-only Stream
+	//Searches for the stream in the packs from the mountpoint in top to bottom order, and opens if found
+	virtual OpenSourceResult openSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const = 0;
+	virtual OpenStreamSourceResult openStreamSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const = 0;
+	virtual OpenMapSourceResult openMapSource(const AbsPath& path, MapView mapView = MapView(), SourceFlags flags = SourceFlags()) const = 0;
+
+	using SinkFlag = ISink::Flag;
+	using SinkFlags = ISink::Flags;
+
+	//Opens a Write-only Stream
+	//Searches for the stream in the packs from the mountpoint in top to bottom order, and opens if found.
+	//If used with the CREATE flag, it will create the stream if not found.
+	//NOTE: this will fail if the mountpoint has only read-only packs (zip files, etc)
+	virtual OpenSinkResult openSink(const AbsPath& path, Mode mode, SinkFlags flags = SinkFlags()) = 0;
+	virtual OpenStreamSinkResult openStreamSink(const AbsPath& path, Mode mode, SinkFlags flags = SinkFlags()) = 0;
+	virtual OpenMapSinkResult openMapSink(const AbsPath& path, Mode mode, size_t size, SinkFlags flags = SinkFlags()) = 0;
+
+	//////////////////////////////////////////////////////////////////////////
+
+	//Returns the physical location of the virtual pack. 
+	virtual ConvertToNativePathResult convertToNativePath(const AbsPath& path) const = 0;
+
+	virtual IsFileResult isFile(const AbsPath& path) const = 0;
+	virtual IsFolderResult isFolder(const AbsPath& path) const = 0;
+	virtual ExistsResult exists(const AbsPath& path) const = 0;
+
+	virtual GetStatResult getStat(const AbsPath& path) const = 0;
+
+	virtual MakeFolderResult makeFolder(const AbsPath& path) = 0;
+
+	virtual RenameResult rename(const AbsPath& path, const AbsPath& newPath) = 0;
+
+	//this removes one file or one folder. The folder is removed ONLY if it's empty. If you want to remove a non-empty folder, use RemoveRecursively
+	virtual RemoveResult remove(const AbsPath& path) = 0;
+
+	virtual CopyResult copy(const AbsPath& path, const AbsPath& newPath) = 0;
+
+	// Hard links are essentially a new name for the same file data.
+	// The file becomes reference counted and the data will be deleted when all handles to it are removed from the FS.
+	virtual MakeHardLinkResult makeHardLink(const AbsPath& sourcePath, const AbsPath& linkPath) = 0;
+
+	// Soft links are like 'weak' pointers to a file. If the original gets deleted the soft link becomes 'null'.
+	virtual MakeSoftLinkResult makeSymLink(const AbsPath& sourcePath, const AbsPath& linkPath) = 0;
+
+	//Use this function in order to remove the set folder and all of the files and folders contained inside.
+	//This function is not atomic; if the removal of any of the folders or files fails, the process will be stopped
+	//and the function will return false, but any of the previously removed files will be already removed.
+	virtual RemoveRecursivelyResult removeRecursively(const AbsPath& path) = 0;
+
+	virtual SetWriteTimeResult setWriteTime(const AbsPath& path, time_t time) = 0;
+
+	virtual cppcoro::generator<EnumerateEntry> enumerate(const AbsPath& path) const = 0;
+	virtual cppcoro::generator<EnumerateEntry> enumerateRecursively(const AbsPath& path) const = 0;
+};
+
+
+}
+

include/fs/IMapSink.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "ISink.h"
+#include "fs/Api.h"
+
+namespace fs
+{
+//Design rationale:
+// Sinks are throw-away objects representing a data source or sink (the Sink).
+// They are not supposed to be reused. 
+// There is only one owner of the sink, enforced by the tl::unique_ref wrapper. Due to this, sinks are NOT thread safe.
+// There is no open/close for sinks as the open is done in the constructor and close in the destructor. This is to reinforce the idea of throw-away objects.
+// If you need a persistent sink to be able to reload your data, hold a 
+//		path to the sink instead of the sink. This is to allow the underlying file system to change (due to DLC for example). When you need the sink again, request a new one from the file system.
+
+class FS_API IMapSink : public ISink
+{
+public:
+	IMapSink() = default;
+	~IMapSink() override = default;
+
+	IMapSink(IMapSink&&) = default;
+	IMapSink& operator=(IMapSink&&) = default;
+
+	//this maps the sink from the current pointer and size bytes. the pointer is advanced with size bytes
+	virtual tl::span<uint8_t> map(size_t size) = 0;
+};
+
+
+}
+
+
+

include/fs/IMapSource.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "ISource.h"
+#include "fs/Api.h"
+#include "tl/span.h"
+
+namespace fs
+{
+
+//Design rationale:
+// Sources are throw-away objects representing a data source or sink (the Sink).
+// They are not supposed to be reused. 
+// There is only one owner of the source, enforced by the tl::unique_ref wrapper. Due to this, sources are NOT thread safe.
+// There is no open/close for sources as the open is done in the constructor and close in the destructor. This is to reinforce the idea of throw-away objects.
+// If you need a persistent source to be able to reload your data, hold a 
+//		path to the source instead of the source. This is to allow the underlying file system to change (due to DLC for example). When you need the source again, request a new one from the file system.
+
+class FS_API IMapSource : public ISource
+{
+public:
+	IMapSource() = default;
+	~IMapSource() override = default;
+
+	IMapSource(IMapSource&&) = default;
+	IMapSource& operator=(IMapSource&&) = default;
+
+	//this maps the source from the current pointer and size bytes. the pointer is advanced with size bytes
+	virtual tl::span<const uint8_t> map(size_t size) = 0;
+};
+
+
+}

include/fs/IPack.h

@@ -0,0 +1,45 @@
+#pragma once
+
+//#pragma warning( disable : 4250)
+
+#include "fs/IFilesystem.h"
+#include "fs/AbsPath.h"
+#include "fs/Error.h"
+#include "fs/Mode.h"
+
+#include "fs/Api.h"
+
+namespace fs
+{
+class ISource;
+class IMapSource;
+
+class FS_API IPack
+{
+public:
+	virtual ~IPack() = default;
+
+	using SourceFlag = IFilesystem::SourceFlag;
+	using SourceFlags = IFilesystem::SourceFlags;
+	using MapView = IFilesystem::MapView;
+
+	virtual OpenSourceResult openSource(const AbsPath& i_path, SourceFlags i_flags = SourceFlags()) const = 0;
+	virtual OpenStreamSourceResult openStreamSource(const AbsPath& i_path, SourceFlags i_flags = SourceFlags()) const = 0;
+	virtual OpenMapSourceResult openMapSource(const AbsPath& i_path, MapView i_mapView = MapView(), SourceFlags i_flags = SourceFlags()) const = 0;
+
+	virtual IsFileResult isFile(const AbsPath& i_path) const = 0;
+	virtual IsFolderResult isFolder(const AbsPath& i_path) const = 0;
+	virtual ExistsResult exists(const AbsPath& i_path) const = 0;
+
+	virtual GetStatResult getStat(const AbsPath& i_path) const = 0;
+
+	virtual cppcoro::generator<EnumerateEntry> enumerate(const AbsPath& i_path) const = 0;
+	virtual cppcoro::generator<EnumerateEntry> enumerateRecursively(const AbsPath& i_path) const = 0;
+
+	virtual ConvertToNativePathResult convertToNativePath(const AbsPath& i_path) const = 0;
+
+protected:
+	IPack() = default;
+};
+
+}

include/fs/ISink.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "fs/IStreamSink.h"
+#include "tl/flag_set2.h"
+#include "fs/Api.h"
+
+namespace fs
+{
+
+class FS_API ISink : public IStreamSink
+{
+public:
+	enum class Flag
+	{
+	};
+	typedef tl::flag_set2<Flag> Flags;
+
+	ISink() = default;
+	~ISink() override = default;
+	
+	ISink(ISink&&) = default;
+	ISink& operator=(ISink&&) = default;
+
+	//changes the sink cursor either to an absolute offset or relative one
+	//NOTE!!! if the cursor is moved beyond the sink size, the size is increased to fit it.
+	virtual void seekBeg(uint64_t offset) = 0;
+	virtual void seekRel(int64_t offset) = 0;
+
+	//returns the sink cursor. Always between 0 and size
+	virtual uint64_t tell() const = 0;
+};
+
+}

include/fs/ISource.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include "fs/IStreamSource.h"
+#include "fs/Api.h"
+#include "tl/flag_set2.h"
+
+namespace fs
+{
+
+//Design rationale:
+// Sources are throw-away objects representing a data source or sink (the Sink).
+// They are not supposed to be reused. 
+// There is only one owner of the source, enforced by the tl::unique_ref wrapper. Due to this, sources are NOT thread safe.
+// There is no open/close for sources as the open is done in the constructor and close in the destructor. This is to reinforce the idea of throw-away objects.
+// If you need a persistent source to be able to reload your data, hold a 
+//		path to the source instead of the source. This is to allow the underlying file system to change (due to DLC for example). When you need the source again, request a new one from the file system.
+
+class FS_API ISource : public IStreamSource
+{
+public:
+	enum class Flag : uint8_t
+	{
+		Unbuffered,
+		Uncached,
+		SequentialHint, //takes prio over the random hint flag
+		RandomHint,
+	};
+	typedef tl::flag_set2<Flag> Flags;
+
+	ISource() = default;
+	~ISource() override = default;
+
+	ISource(ISource&&) = default;
+	ISource& operator=(ISource&&) = default;
+
+	//Changes the cursor either to an absolute offset or a relative one
+	virtual void seekBeg(uint64_t offset) = 0;
+	virtual void seekRel(int64_t offset) = 0;
+
+	//returns the current cursor position. This is always between 0 and size
+	virtual uint64_t tell() const = 0;
+
+	//returns the size of the source in bytes
+	virtual uint64_t getSize() const = 0;
+};
+
+}

include/fs/IStreamSink.h

@@ -0,0 +1,92 @@
+#pragma once
+
+#include <tl/optional.h>
+#include <tl/span.h>
+#include "fs/Error.h"
+#include "fs/Api.h"
+
+namespace fs
+{
+
+class FS_API IStreamSink
+{
+public:
+	IStreamSink() = default;
+	virtual ~IStreamSink() = default;
+
+	IStreamSink(IStreamSink&&) = default;
+	IStreamSink& operator=(IStreamSink&&) = default;
+
+	//This will return the last generated error AND CLEAR IT AFTERWARDS.
+	virtual tl::optional<Error> getLastError() const = 0;
+
+	//writes size bytes from the buffer to the sink at the current cursor position.
+	//the sink size is increased if the sinks reaches the end
+	virtual size_t write(tl::span<const uint8_t> data) = 0;
+
+	//returns the size of the sink. Always >= tell
+	virtual uint64_t getSize() const = 0;
+
+	// The preferred buffer size for writes. Use this buffer size for optimal performance
+	// If zero, use whatever you want
+	virtual size_t getPreferredBufferSize() const = 0;
+};
+
+template<typename T, bool isEnum>
+struct StreamSinkSerializationHelperWrite_Internal;
+
+template<typename T>
+struct StreamSinkSerializationHelperWrite_Internal<T, false>
+{
+	static IStreamSink& write(IStreamSink& s, const T& val)
+	{
+		static_assert(std::is_standard_layout_v<T> && !std::is_pointer_v<T> && !std::is_array_v<T>, "You need a specialized write for T");
+		s.write({ reinterpret_cast<const uint8_t*>(&val), sizeof(T) });
+		return s;
+	}
+
+	static IStreamSink& write(IStreamSink& s, tl::span<const T> val)
+	{
+		const uint32_t count = static_cast<uint32_t>(val.size());
+		s.write({ reinterpret_cast<const uint8_t*>(&count), sizeof(count) });
+		for(const T& v : val)
+			s.write( { reinterpret_cast<const uint8_t*>(&v), sizeof(T) });
+		return s;
+	}
+};
+
+// specialization to read enums
+template<typename EnumType>
+struct StreamSinkSerializationHelperWrite_Internal<EnumType, true>
+{
+	static IStreamSink& write(IStreamSink& s, const EnumType& val)
+	{
+		// first read into the temporary of the underlying type
+		using underlying_t = std::underlying_type_t<EnumType>;
+		underlying_t underlyingValue = static_cast<underlying_t>(val);
+
+		IStreamSink& ret = StreamSinkSerializationHelperWrite_Internal<underlying_t, false>::write(s, underlyingValue);
+		return ret;
+	}
+};
+
+template<typename T>
+struct SimpleStreamSerializationHelperWrite : StreamSinkSerializationHelperWrite_Internal<T, std::is_enum_v<std::remove_reference_t<T>>>
+{
+};
+
+template <typename T>
+IStreamSink& operator<<(IStreamSink& s, const T& val)
+{
+	return SimpleStreamSerializationHelperWrite<T>::write(s, val);
+}
+
+template <typename T>
+IStreamSink& operator<<(IStreamSink& s, tl::span<const T> val)
+{
+	return SimpleStreamSerializationHelperWrite<T>::write(s, val);
+}
+
+}
+
+

include/fs/IStreamSource.h

@@ -0,0 +1,101 @@
+#pragma once
+
+#include "tl/memory_buffer.h"
+#include <tl/optional.h>
+#include <tl/span.h>
+#include "tl/result.h"
+#include "fs/Error.h"
+#include "fs/Api.h"
+
+
+namespace fs
+{
+
+//Design rationale:
+// Throw-away objects representing a non-seekable, non-sized data source or sink.
+// They are not supposed to be reused. 
+// There is only one owner of the source, enforced by the tl::unique_ref wrapper. Due to this, simple sources are NOT thread safe.
+// There is no open/close for simple sources as the open is done in the constructor and close in the destructor. This is to reinforce the idea of throw-away objects.
+// If you need a persistent source to be able to reload your data, hold a 
+//		path to the source instead of the source. This is to allow the underlying file system to change (due to DLC for example). When you need the source again, request a new one from the file system.
+
+class FS_API IStreamSource
+{
+public:
+	IStreamSource() = default;
+	virtual ~IStreamSource() = default;
+
+	IStreamSource(IStreamSource&&) = default;
+	IStreamSource& operator=(IStreamSource&&) = default;
+
+	//This will return the last generated error AND CLEAR IT AFTERWARDS.
+	virtual tl::optional<Error> getLastError() const = 0;
+
+	//Reads from the source into the buffer size bytes. The source pointer is advanced
+	//Returns the number of bytes successfully read
+	virtual size_t read(tl::span<uint8_t> data) = 0;
+
+	virtual bool isEOS() const = 0;
+
+	// The preferred buffer size for reads. Use this buffer size for optimal performance
+	// If zero, use whatever you want
+	virtual size_t getPreferredBufferSize() const = 0;
+};
+
+template<typename T, bool isEnum>
+struct StreamSourceSerializationHelper_Internal;
+
+template<typename T>
+struct StreamSourceSerializationHelper_Internal<T, false>
+{
+	static IStreamSource& read(IStreamSource& s, T& val)
+	{
+		static_assert(std::is_fundamental_v<T> && !std::is_pointer_v<T> && !std::is_array_v<T>, "You need a specialized read for T");
+		s.read({ reinterpret_cast<uint8_t*>(&val), sizeof(T) });
+		return s;
+	}
+
+	static IStreamSource& read(IStreamSource& s, tl::memory_buffer& val)
+	{
+		uint32_t count = 0;
+		s.read({ reinterpret_cast<uint8_t*>(&count), sizeof(count) });
+		val.resize(count);
+		s.read({ val.data(), count });
+		return s;
+	}
+};
+
+// specialization to read enums
+template<typename EnumType>
+struct StreamSourceSerializationHelper_Internal<EnumType, true>
+{
+	static IStreamSource& read(IStreamSource& s, EnumType& val)
+	{
+		// first read into the temporary of the underlying type
+		using underlying_t = std::underlying_type_t<EnumType>;
+		underlying_t underlyingValue;
+
+		IStreamSource& ret = StreamSourceSerializationHelper_Internal<underlying_t, false>::read(s, underlyingValue);
+		// put the read value into the enum
+		val = EnumType(underlyingValue);
+		return ret;
+	}
+};
+
+template<typename T>
+struct SimpleStreamSerializationHelper : StreamSourceSerializationHelper_Internal<T, std::is_enum_v<std::remove_reference_t<T>>>
+{
+};
+
+template <typename T>
+IStreamSource& operator>>(IStreamSource& s, T& val)
+{
+	return SimpleStreamSerializationHelper<T>::read(s, val);
+}
+
+inline IStreamSource& operator>>(IStreamSource& s, tl::memory_buffer& val)
+{
+	return SimpleStreamSerializationHelper<uint8_t>::read(s, val);
+}
+
+}

include/fs/IWritablePack.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "IPack.h"
+#include "Mode.h"
+#include "fs/Api.h"
+
+namespace fs
+{
+
+class FS_API IWritablePack : virtual public IPack
+{
+public:
+	using SinkFlag = IFilesystem::SinkFlag;
+	using SinkFlags = IFilesystem::SinkFlags;
+
+	virtual OpenSinkResult openSink(const AbsPath& i_path, Mode i_mode, SinkFlags i_flags = SinkFlags()) = 0;
+	virtual OpenStreamSinkResult openStreamSink(const AbsPath& i_path, Mode i_mode, SinkFlags i_flags = SinkFlags()) = 0;
+	virtual OpenMapSinkResult openMapSink(const AbsPath& i_path, Mode i_mode, size_t i_size, SinkFlags i_flags = SinkFlags()) = 0;
+
+	virtual RemoveResult remove(const AbsPath& i_path) = 0;
+	virtual RenameResult rename(const AbsPath& i_path, const AbsPath& i_newPath) = 0;
+
+	virtual MakeFolderResult makeFolder(const AbsPath& i_path) = 0;
+	virtual RemoveRecursivelyResult removeRecursively(const AbsPath& i_path) = 0;
+
+	virtual SetWriteTimeResult setWriteTime(const AbsPath& i_path, time_t i_time) = 0;
+};
+
+}

include/fs/MapSourceView.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include "tl/ptr.h"
+#include "IMapSource.h"
+#include "MemoryViewSource.h"
+#include "fs/Api.h"
+
+namespace fs
+{
+
+class FS_API MapSourceView final : public IMapSource
+{
+public:
+
+	MapSourceView(tl::lent_ref<IMapSource> srcSource, uint64_t offset, size_t size);
+
+	tl::optional<Error> getLastError() const override;
+
+	size_t read(tl::span<uint8_t> data) override;
+
+	void seekBeg(uint64_t offset) override;
+	void seekRel(int64_t offset) override;
+
+	uint64_t tell() const override;
+	uint64_t getSize() const override;
+
+	tl::span<const uint8_t> map(size_t size) override;
+
+	bool isEOS() const override;
+
+	size_t getPreferredBufferSize() const override;
+
+private:
+
+	tl::span<const uint8_t> m_memView;
+	MemoryViewSource m_memViewSource;
+
+	tl::span<const uint8_t> constructMemView(IMapSource& srcSource, uint64_t offset, size_t size) const;
+};
+
+}
+

include/fs/MemorySink.h

@@ -0,0 +1,56 @@
+#pragma once
+
+#include "fs/IMapSink.h"
+#include "fs/Error.h"
+#include "tl/memory_buffer.h"
+#include "fs/Api.h"
+
+namespace fs
+{
+//////////////////////////////////////////////////////////////////////////
+
+class MemorySource;
+
+class FS_API MemorySink final : public IMapSink
+{
+public:
+	MemorySink(size_t reserve = BUFSIZ);
+	explicit MemorySink(tl::memory_buffer&& buffer);
+	~MemorySink() override = default;
+
+	MemorySink(MemorySink&& sink) = default;
+	MemorySink& operator=(MemorySink&& sink) = default;
+
+	tl::optional<Error> getLastError() const override;
+
+	size_t write(tl::span<const uint8_t> data) override;
+	void seekBeg(uint64_t offset) override;
+	void seekRel(int64_t offset) override;
+	uint64_t tell() const override;
+	uint64_t getSize() const override;
+	tl::span<const uint8_t> getDataView() const;
+	tl::memory_buffer getDataCopy() const;
+	tl::memory_buffer getDataMove();
+	void clear();
+	void swap(MemorySource& source);
+	void reserve(size_t capacity);
+	void shrinkToFit();
+
+	tl::span<uint8_t> map(size_t size) override;
+
+	size_t getPreferredBufferSize() const override;
+
+	//this applies OVER the current size
+	void preReserve(size_t size);
+
+	static tl::memory_buffer extractData(MemorySink sink);
+
+private:
+	size_t m_offset = 0;
+	tl::memory_buffer m_data;
+	mutable tl::optional<Error> m_optLastError;
+};
+
+//////////////////////////////////////////////////////////////////////////
+}
+

include/fs/MemorySource.h

@@ -0,0 +1,55 @@
+#pragma once
+
+#include "IMapSource.h"
+#include "tl/memory_buffer.h"
+#include "fs/Api.h"
+
+namespace fs
+{
+class MemorySink;
+
+class FS_API MemorySource final : public IMapSource
+{
+	friend class MemorySink;
+public:
+	// The source will make an internal copy of the data passed by the constructor
+	// rawMemory can be safely destroyed after calling the constructor
+	explicit MemorySource(tl::span<const uint8_t> data);
+	explicit MemorySource(MemorySink& sink);
+	explicit MemorySource(const tl::memory_buffer& memoryBuffer);
+
+	MemorySource(MemorySource&&) = default;
+	MemorySource& operator=(MemorySource&&) = default;
+	
+	//we have another constructor for empty source
+	MemorySource() = default;
+	~MemorySource() override = default;
+
+	tl::optional<Error> getLastError() const override;
+
+	size_t read(tl::span<uint8_t> data) override;
+
+	void seekBeg(uint64_t offset) override;
+	void seekRel(int64_t offset) override;
+
+	uint64_t tell() const override;
+	uint64_t getSize() const override;
+
+	tl::span<const uint8_t> map(size_t size) override;
+
+	void swap(MemorySink& sink);
+
+	bool isEOS() const override;
+
+	size_t getPreferredBufferSize() const override;
+
+protected:
+	tl::memory_buffer m_data;
+
+private:
+	size_t m_offset = 0;
+	mutable tl::optional<Error> m_optLastError;
+};
+
+}
+

include/fs/MemoryViewSource.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include "IMapSource.h"
+#include "fs/Api.h"
+#include "tl/optional.h"
+
+namespace fs
+{
+
+class FS_API MemoryViewSource final : public IMapSource
+{
+public:
+	~MemoryViewSource() override = default;
+
+	explicit MemoryViewSource(tl::span<const uint8_t> memView);
+
+	MemoryViewSource(MemoryViewSource&&) = default;
+	MemoryViewSource& operator=(MemoryViewSource&&) = default;
+
+	tl::optional<Error> getLastError() const override;
+
+	size_t read(tl::span<uint8_t> data) override;
+
+	void seekBeg(uint64_t offset) override;
+	void seekRel(int64_t offset) override;
+
+	uint64_t tell() const override;
+	uint64_t getSize() const override;
+
+	tl::span<const uint8_t> map(size_t size) override;
+
+	bool isEOS() const override;
+
+	size_t getPreferredBufferSize() const override;
+
+protected:
+
+	tl::span<const uint8_t> m_memView;
+
+private:
+	uint64_t m_offset = 0;
+	mutable tl::optional<Error> m_optLastError;
+
+};
+
+}
+

include/fs/Mode.h

@@ -0,0 +1,16 @@
+#pragma once
+
+namespace fs
+{
+
+enum class Mode
+{
+	CreateOrOpen,			//Create if doesn't exist, or open otherwise, but DON'T clear the file
+	CreateOrOpenAndClear,	//Create if doesn't exist, or open otherwise, and clear the file
+	CreateIfNew,			//Create the file ONLY if it doesn't exist already. Otherwise FAIL
+
+	Open,					//Open if exists, fail otherwise, but DON'T clear the file
+	OpenAndClear			//Open the file if exists, fail otherwise, and clear the file
+};
+
+}

include/fs/NativeFilesystem.h

@@ -0,0 +1,82 @@
+#pragma once
+
+#include "IFilesystem.h"
+#include "fs/Error.h"
+#include "fs/Mode.h"
+#include "fs/Api.h"
+#include "tl/vector.h"
+
+namespace fs
+{
+class FS_API NativeFilesystem final : public IFilesystem, public tl::lendable_base<NativeFilesystem>
+{
+public:
+
+	enum class CheckFlag : uint8_t
+	{
+		ValidatePathCase, //Win32 only, relatively slow, but makes the filesystem behave more like on posix (case-sensitive filenames)
+	};
+	using CheckFlags = tl::flag_set2<CheckFlag>;
+
+	NativeFilesystem() = default;
+	explicit NativeFilesystem(CheckFlags checkFlags);
+
+	OpenSourceResult openSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override;
+	OpenStreamSourceResult openStreamSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override;
+	OpenMapSourceResult openMapSource(const AbsPath& path, MapView mapView = MapView(), SourceFlags flags = SourceFlags()) const override;
+	OpenSinkResult openSink(const AbsPath& path, Mode mode, SinkFlags flags = SinkFlags()) override;
+	OpenStreamSinkResult openStreamSink(const AbsPath& path, Mode mode, SinkFlags flags = SinkFlags()) override;
+	OpenMapSinkResult openMapSink(const AbsPath& path, Mode mode, size_t size, SinkFlags flags = SinkFlags()) override;
+
+	ConvertToNativePathResult convertToNativePath(const AbsPath& path) const override;
+
+	IsFileResult isFile(const AbsPath& path) const override;
+	IsFolderResult isFolder(const AbsPath& path) const override;
+	ExistsResult exists(const AbsPath& path) const override;
+
+	MakeFolderResult makeFolder(const AbsPath& path) override;
+
+	RenameResult rename(const AbsPath& path, const AbsPath& newPath) override;
+
+	RemoveResult remove(const AbsPath& path) override;
+	RemoveRecursivelyResult removeRecursively(const AbsPath& path) override;
+
+	CopyResult copy(const AbsPath& path, const AbsPath& newPath) override;
+
+	MakeHardLinkResult makeHardLink(const AbsPath& sourcePath, const AbsPath& linkPath) override;
+	MakeSoftLinkResult makeSymLink(const AbsPath& sourcePath, const AbsPath& linkPath) override;
+
+	SetWriteTimeResult setWriteTime(const AbsPath& path, time_t time) override;
+
+	GetStatResult getStat(const AbsPath& path) const override;
+
+	cppcoro::generator<EnumerateEntry> enumerate(const AbsPath& path) const override;
+	cppcoro::generator<EnumerateEntry> enumerateRecursively(const AbsPath& path) const override;
+
+	AbsPath getCurrentFolder() const;
+	tl::result<Error> setCurrentFolder(const AbsPath& path);
+
+	typedef tl::result<tl::vector<tl::string>, Error> EnumerateFilesResult;
+	EnumerateFilesResult enumerateFiles(const AbsPath& fullPath) const;
+
+	typedef tl::result<tl::vector<tl::string>, Error> EnumerateFoldersResult;
+	EnumerateFoldersResult enumerateFolders(const AbsPath& fullPath) const;
+
+	AbsPath makeAbsPath(const tl::string& string) const;
+	
+
+private:
+	CheckFlags m_checkFlags;
+};
+
+//this FS will do various checks to make the platforms behave similarly
+//For example path case checks on Win32 to make its FS case-sensitive
+FS_API extern NativeFilesystem native;
+
+//this FS is RAW with no advanced (read: slow) checks
+//Use it where appropriate, where native is clearly slowing down something vital.
+//Normal use-case: tools
+//*** For game or library use, consult with the lead, and think 3 times. ***
+FS_API extern NativeFilesystem nativeRaw;
+
+}

include/fs/NativeUtils.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <EASTL/fixed_string.h>
+#include "AbsPath.h"
+
+namespace fs
+{
+namespace native_utils
+{
+
+bool validateCaseSensitiveFilename(const AbsPath& filePath) noexcept;
+bool validateCaseSensitiveFolder(const AbsPath& folderPath) noexcept;
+
+typedef eastl::fixed_string<wchar_t, 1024> wstring_t;
+typedef eastl::fixed_string<char, 1024> string_t;
+
+wstring_t utf8To16(tl::span<const char> str) noexcept;
+string_t utf16To8(tl::span<const wchar_t> str) noexcept;
+
+////////////////////////////////////////////////////////////////////////
+
+}
+}

include/fs/ProcessStream.h

@@ -0,0 +1,14 @@
+#pragma once
+#include "Error.h"
+#include "fs/Api.h"
+#include "tl/result.h"
+
+namespace fs
+{
+class IStreamSource;
+
+typedef tl::result<void, Error> ProcessStreamResult;
+typedef tl::function<void(tl::span<const uint8_t> data)> ProcessDataCallback;
+// This API Reads the provided stream and pipes it through the provided data callback to allow implementation of custom filters/encodings.
+FS_API ProcessStreamResult processStream(IStreamSource& source, const ProcessDataCallback& dataCallback, size_t bufferSize = 0);
+}

include/fs/ReadStream.h

@@ -0,0 +1,20 @@
+#pragma once
+#include "Error.h"
+#include "tl/memory_buffer.h"
+#include "fs/Api.h"
+#include "IFilesystem.h"
+
+namespace fs
+{
+class IStreamSource;
+
+typedef tl::result<void, Error> ReadStreamResult;
+FS_API ReadStreamResult readStream(tl::memory_buffer& buffer, IStreamSource& source, size_t bufferSize = 0);
+FS_API ReadStreamResult readStream(std::string& buffer, IStreamSource& source, size_t bufferSize = 0);
+FS_API ReadStreamResult readStream(eastl::string& buffer, IStreamSource& source, size_t bufferSize = 0);
+
+FS_API ReadStreamResult readFile(tl::memory_buffer& buffer, IFilesystem& filesystem, const AbsPath& filePath, size_t bufferSize = 0);
+FS_API ReadStreamResult readFile(std::string& buffer, IFilesystem& filesystem, const AbsPath& filePath, size_t bufferSize = 0);
+FS_API ReadStreamResult readFile(eastl::string& buffer, IFilesystem& filesystem, const AbsPath& filePath, size_t bufferSize = 0);
+
+}

include/fs/SourceView.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "ISource.h"
+#include <tl/ptr.h>
+#include "fs/Api.h"
+
+namespace fs
+{
+
+class FS_API SourceView final : public ISource
+{
+public:
+	SourceView(tl::unique_ref<ISource>&& i_srcSource, uint64_t i_offset, uint64_t i_size);
+	~SourceView() override = default;
+
+	tl::optional<Error> getLastError() const override;
+
+	size_t read(tl::span<uint8_t> data) override;
+
+	void seekBeg(uint64_t i_offset) override;
+	void seekRel(int64_t i_offset) override;
+
+	uint64_t tell() const override;
+	uint64_t getSize() const override;
+	
+	bool isEOS() const override;
+
+	size_t getPreferredBufferSize() const override;
+
+private:
+	tl::unique_ref<ISource> m_srcSource;
+	mutable tl::optional<Error> m_optLastError;
+	uint64_t	m_viewOffset = 0;
+	uint64_t	m_viewSize = 0;
+	uint64_t	m_offset = 0;
+};
+
+}
+

include/fs/StreamSourceToSourceAdapter.h

@@ -0,0 +1,43 @@
+#pragma once
+#include "IMapSource.h"
+#include "IStreamSource.h"
+#include "tl/memory_buffer.h"
+#include "fs/Api.h"
+#include "tl/ptr.h"
+
+namespace fs
+{
+
+class FS_API StreamSourceToSourceAdapter final : public IMapSource
+{
+public:
+	explicit StreamSourceToSourceAdapter(tl::unique_ref<IStreamSource>&& i_srcSource);
+	~StreamSourceToSourceAdapter() override = default;
+
+	tl::optional<Error> getLastError() const override;
+
+	size_t read(tl::span<uint8_t> data) override;
+
+	void seekBeg(uint64_t i_offset) override;
+	void seekRel(int64_t i_offset) override;
+
+	uint64_t tell() const override;
+	uint64_t getSize() const override;
+
+	bool isEOS() const override;
+
+	size_t getPreferredBufferSize() const override;
+
+	tl::span<const uint8_t> map(size_t size) override;
+
+private:
+	void readAll() const;
+
+	tl::unique_ref<IStreamSource> m_srcSource;
+	mutable tl::optional<Error> m_optLastError;
+	mutable tl::memory_buffer m_data;
+	mutable bool m_readAll = false;
+	uint64_t	m_offset = 0;
+};
+
+}

include/fs/WritableFolderPack.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "fs/IWritablePack.h"
+#include "fs/FolderPack.h"
+#include "fs/Error.h"
+#include "fs/Mode.h"
+#include "fs/Api.h"
+
+namespace fs
+{
+
+class FS_API WritableFolderPack final : virtual public IWritablePack, public FolderPack
+{
+public:
+	explicit WritableFolderPack(AbsPath location);
+	WritableFolderPack(tl::lent_ref<IFilesystem> filesystem, AbsPath location);
+	~WritableFolderPack() override = default;
+
+	OpenSinkResult openSink(const AbsPath& i_path, Mode i_mode, SinkFlags i_flags = SinkFlags()) override;
+	OpenStreamSinkResult openStreamSink(const AbsPath& i_path, Mode i_mode, SinkFlags i_flags = SinkFlags()) override;
+	OpenMapSinkResult openMapSink(const AbsPath& i_path, Mode i_mode, size_t i_size, SinkFlags i_flags = SinkFlags()) override;
+
+	RemoveResult remove(const AbsPath& i_path) override;
+	RenameResult rename(const AbsPath& i_path, const AbsPath& i_newPath) override;
+
+	MakeFolderResult makeFolder(const AbsPath& i_path) override;
+	RemoveRecursivelyResult removeRecursively(const AbsPath& i_path) override;
+
+	SetWriteTimeResult setWriteTime(const AbsPath& i_path, time_t i_time) override;
+};
+
+}

include/fs/zip/DeflateFileWriter.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include "fs/AbsPath.h"
+#include "fs/zip/ZipWriter.h"
+
+#include "tl/ptr.h"
+
+namespace fs
+{
+
+class IFilesystem;
+
+//////////////////////////////////////////////////////////////////////////
+
+class FS_API DeflateFileWriter
+{
+public:
+	//compression level: 0 to 9
+	DeflateFileWriter(AbsPath i_filePath, tl::lent_ref<IFilesystem> i_filesystem, uint8_t i_compressionLevel);
+
+	DeflateFileWriter(const DeflateFileWriter&) = delete;
+	DeflateFileWriter& operator=(const DeflateFileWriter&) = delete;
+
+	DeflateFileWriter(DeflateFileWriter&&) = default;
+	DeflateFileWriter& operator=(DeflateFileWriter&&) = default;
+
+	ZipWriter::DataWriterResult operator()(IStreamSink& i_sink);
+
+private:
+	AbsPath m_filePath;
+	tl::lent_ref<IFilesystem> m_filesystem;
+	uint8_t m_compressionLevel = 9;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+}

include/fs/zip/DeflateSinkWriter.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include "fs/IStreamSource.h"
+#include "fs/zip/ZipWriter.h"
+
+namespace fs
+{
+
+//////////////////////////////////////////////////////////////////////////
+
+class ISource;
+//////////////////////////////////////////////////////////////////////////
+
+class FS_API DeflateSinkWriter
+{
+public:
+	//compression level: 0 to 9
+	DeflateSinkWriter(IStreamSource& source, uint8_t compressionLevel);
+	DeflateSinkWriter(tl::span<const uint8_t> data, uint8_t compressionLevel);
+
+	DeflateSinkWriter(const DeflateSinkWriter&) = delete;
+	DeflateSinkWriter& operator=(const DeflateSinkWriter&) = delete;
+
+	DeflateSinkWriter(DeflateSinkWriter&&) = default;
+	DeflateSinkWriter& operator=(DeflateSinkWriter&&) = default;
+
+	ZipWriter::DataWriterResult operator()(IStreamSink& sink);
+
+private:
+	IStreamSource* m_source = nullptr;
+	tl::span<const uint8_t> m_data;
+	uint8_t m_compressionLevel = 9;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+}

include/fs/zip/StoreFileWriter.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "fs/AbsPath.h"
+#include "fs/zip/ZipWriter.h"
+
+#include "tl/ptr.h"
+
+namespace fs
+{
+
+class IFilesystem;
+
+//////////////////////////////////////////////////////////////////////////
+
+class FS_API StoreFileWriter
+{
+public:
+	StoreFileWriter(AbsPath i_filePath, tl::lent_ref<IFilesystem> i_filesystem);
+
+	StoreFileWriter(const StoreFileWriter&) = delete;
+	StoreFileWriter& operator=(const StoreFileWriter&) = delete;
+
+	StoreFileWriter(StoreFileWriter&&) = default;
+	StoreFileWriter& operator=(StoreFileWriter&&) = default;
+
+	ZipWriter::DataWriterResult operator()(IStreamSink& i_sink);
+
+private:
+	AbsPath m_filePath;
+	tl::lent_ref<IFilesystem> m_filesystem;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+}

include/fs/zip/StoreSinkWriter.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "fs/zip/ZipWriter.h"
+
+namespace fs
+{
+
+//////////////////////////////////////////////////////////////////////////
+
+class ISource;
+//////////////////////////////////////////////////////////////////////////
+
+class FS_API StoreSinkWriter
+{
+public:
+	explicit StoreSinkWriter(ISource& source);
+
+	StoreSinkWriter(const StoreSinkWriter&) = delete;
+	StoreSinkWriter& operator=(const StoreSinkWriter&) = delete;
+
+	StoreSinkWriter(StoreSinkWriter&&) = default;
+	StoreSinkWriter& operator=(StoreSinkWriter&&) = default;
+
+	ZipWriter::DataWriterResult operator()(IStreamSink& sink);
+
+private:
+	tl::reference_wrapper<ISource> m_inSource;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+}

include/fs/zip/ZipBase.h

@@ -0,0 +1,158 @@
+#pragma once
+#include "tl/flag_set.h"
+#include "fs/Api.h"
+
+namespace fs
+{
+
+class FS_API ZipBase
+{
+public:
+	enum class GeneralBitFlag : uint16_t
+	{
+		Encrypted = 1 << 0,
+		CompressionMethodSpecific1 = 1 << 1,
+		CompressionMethodSpecific2 = 1 << 2,
+		DataDescriptorNeeded = 1 << 3,
+		StrongEncryption = 1 << 6,
+		LanguageEncodingFlag = 1 << 11,
+		LocalHeaderFieldsMasked = 1 << 13,
+	};
+	typedef tl::flag_set<GeneralBitFlag> GeneralBitFlags;
+
+	enum class CompressionMethod : uint16_t
+	{
+		Store = 0,
+		Shrunk = 1,
+		Reduced1 = 2,
+		Reduced2 = 3,
+		Reduced3 = 4,
+		Reduced4 = 5,
+		Implode = 6,
+		//Reserved = 7,
+		Deflate = 8,
+		Deflate64 = 9, 
+		PKWareImplode = 10,
+		//Reserved = 11,
+		BZip2 = 12,
+		//Reserved = 13,
+		LZMA = 14,
+		//Reserved = 15,
+		//Reserved = 16,
+		//Reserved = 17,
+		Terse = 18,
+		LZ77 = 19,
+
+		LZ4 = 20, // non-standard
+		ZStd = 21, // non-standard
+
+		WavPack = 97,
+		PPMd = 98,
+	};
+
+protected:
+	//////////////////////////////////////////////////////////////////////////
+	//These structs follow pretty closely the https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT standard
+
+#pragma pack(push, 1)
+
+	struct LocalFileHeader
+	{
+		static constexpr uint32_t k_signature = 0x04034b50u;
+		uint32_t signature = k_signature;
+		uint16_t versionNeededToExtract = 0;
+		uint16_t generalBitFlag = 0;
+		uint16_t compressionMethod = 0;
+		uint16_t lastModFileTime = 0;
+		uint16_t lastModFileDate = 0;
+		uint32_t crc32 = 0;
+		uint32_t compressedSize = 0;
+		uint32_t uncompressedSize = 0;
+		uint16_t filenameLength = 0;
+		uint16_t extraFieldLength = 0;
+	};
+
+	struct DataDescriptor
+	{
+		uint32_t crc32 = 0;
+		uint32_t compressedSize = 0;
+		uint32_t uncompressedSize = 0;
+	};
+
+	struct CentralDirectoryFileHeader
+	{
+		static constexpr uint32_t k_signature = 0x02014b50u;
+		uint32_t signature = k_signature;
+		uint16_t versionMadeBy = 0;
+		uint16_t versionNeededToExtract = 0;
+		uint16_t generalBitFlag = 0;
+		uint16_t compressionMethod = 0;
+		uint16_t lastModFileTime = 0;
+		uint16_t lastModFileDate = 0;
+		uint32_t crc32 = 0;
+		uint32_t compressedSize = 0;
+		uint32_t uncompressedSize = 0;
+		uint16_t filenameLength = 0;
+		uint16_t extraFieldLength = 0;
+		uint16_t fileCommentLength = 0;
+		uint16_t diskNumberStart = 0;
+		uint16_t internalFileAttributes = 0;
+		uint32_t externalFileAttributes = 0;
+		uint32_t localHeaderOffset = 0;
+	};
+
+	struct EndOfCentralDirectoryRecord
+	{
+		static constexpr uint32_t k_signature = 0x06054b50u;
+		uint32_t signature = k_signature;
+		uint16_t diskNumber = 0;
+		uint16_t centralDirectoryStartDiskNumber = 0;
+		uint16_t centralDirectoryRecordCountOnThisDisk = 0;
+		uint16_t centralDirectoryRecordCount = 0;
+		uint32_t centralDirectorySize = 0;
+		uint32_t centralDirectoryOffset = 0;
+		uint16_t zipFileCommentLength = 0;
+	};
+
+	struct ExtendedInformationExtraField64
+	{
+		static constexpr uint32_t k_tag = 0x0001;
+		uint16_t tag = k_tag;
+		uint16_t size = 0;
+		uint64_t originalSize = 0;
+		uint64_t compressedSize = 0;
+		uint64_t localHeaderOffset = 0;
+		uint32_t diskNumber = 0;
+	};
+
+	struct EndOfCentralDirectoryRecord64
+	{
+		static constexpr uint32_t k_signature = 0x06064b50u;
+		uint32_t signature = k_signature;
+		uint64_t endOfCentralDirectorySize = 0;
+		uint16_t versionMadeBy = 0;
+		uint16_t versionNeededToExtract = 0;
+		uint32_t diskNumber = 0;
+		uint32_t centralDirectoryStartDiskNumber = 0;
+		uint64_t centralDirectoryRecordCountOnThisDisk = 0;
+		uint64_t centralDirectoryRecordCount = 0;
+		uint64_t centralDirectorySize = 0;
+		uint64_t centralDirectoryOffset = 0;
+	};
+
+	struct EndOfCentralDirectoryLocator64
+	{
+		static constexpr uint32_t k_signature = 0x07064b50u;
+		uint32_t signature = k_signature;
+		uint32_t centralDirectoryStartDiskNumber = 0;
+		uint64_t endOfCentralDirectoryRecordOffset = 0;
+		uint32_t diskCount = 0;
+	};
+
+#pragma pack(pop)
+
+	static void computeMsDosTime(time_t time, uint16_t& o_date, uint16_t& o_time);
+	static CentralDirectoryFileHeader computeCentralDirectoryFileHeader(const LocalFileHeader& header, bool zip64);
+};
+
+}

include/fs/zip/ZipPack.h

@@ -0,0 +1,121 @@
+#pragma once
+
+#include "tl/result.h"
+#include "tl/string.h"
+#include "tl/vector_map.h"
+
+#include "fs/IPack.h"
+#include "fs/AbsPath.h"
+#include "fs/zip/ZipReader.h"
+
+#include "fs/Api.h"
+
+namespace fs
+{
+class MapSourceView;
+
+class FS_API ZipPack final : virtual public IPack
+{
+public:
+	ZipPack() = default;
+	~ZipPack() override = default;
+
+	typedef tl::result<tl::unique_ref<ZipPack>, Error> CreateResult;
+
+	static CreateResult create(tl::unique_ref<IMapSource> zipFileSource);
+	static CreateResult create(tl::lent_ref<const IFilesystem> filesystem, AbsPath zipFileLocation);
+
+	virtual void setEncryptionData(const tl::string& key, uint32_t rounds);
+
+	OpenSourceResult openSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override;
+	OpenStreamSourceResult openStreamSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override;
+	OpenMapSourceResult openMapSource(const AbsPath& path, MapView mapView = MapView(), SourceFlags flags = SourceFlags()) const override;
+
+	IsFileResult isFile(const AbsPath& path) const override;
+	IsFolderResult isFolder(const AbsPath& path) const override;
+	ExistsResult exists(const AbsPath& path) const override;
+
+	GetStatResult getStat(const AbsPath& path) const override;
+
+	cppcoro::generator<EnumerateEntry> enumerate(const AbsPath& path) const override;
+	cppcoro::generator<EnumerateEntry> enumerateRecursively(const AbsPath& path) const override;
+
+	tl::result<AbsPath, Error> convertToNativePath(const AbsPath& path) const override;
+
+protected:
+	struct File;
+	struct Folder;
+	struct EntryIndex;
+
+	ZipPack(tl::unique_ref<IMapSource> zipFileSource, ZipReader zipReader);
+	ZipPack(tl::lent_ref<const IFilesystem> filesystem, AbsPath zipFileLocation, ZipReader zipReader);
+
+	OpenMapSourceResult openRawMapSource(const File& file, MapView mapView) const;
+	OpenSourceResult openRawSource(const File& file) const;
+
+	OpenSourceResult openZipSource(const File& file) const;
+	OpenMapSourceResult openZipMapSource(const File& file) const;
+	OpenStreamSourceResult openZipStreamSource(const File& file) const;
+
+	uint16_t getOrAddFolder(AbsPath folderPath, tl::vector<tl::vector<EntryIndex>>& io_childrenIndices);
+	void createEntries(ZipReader zipReader);
+
+	struct File
+	{
+		tl::string name;
+		uint8_t isCompressed : 1;
+		uint8_t uncompressedSizeH = 0;
+		uint8_t compressedSizeH = 0;
+		uint8_t dataOffsetH = 0;
+		uint16_t parentIndex = 0;
+		uint32_t lastModTimePoint = 0;
+		uint32_t uncompressedSizeL = 0;
+		uint32_t compressedSizeL = 0;
+		uint32_t dataOffsetL = 0;
+	};
+	static_assert(sizeof(File) <= sizeof(tl::string) + 40, "Check your sizes!!!");
+
+	struct Folder
+	{
+		tl::string name;
+		uint16_t parentIndex = 0;
+		uint16_t childrenStartIndex = 0; // this is an index inside m_childrenIndices
+		uint16_t childrenCount = 0;
+		uint32_t lastModTimePoint = 0;
+	};
+	static_assert(sizeof(Folder) <= sizeof(tl::string) + 20, "Check your sizes!!!");
+
+	struct EntryIndex
+	{
+		EntryIndex() = default;
+		EntryIndex(bool isFolder, uint16_t index)
+			: isFolder(isFolder)
+			, index(index)
+		{
+		}
+		uint32_t isFolder : 1;
+		uint32_t index : 16;
+	};
+	static_assert(sizeof(EntryIndex) == 4, "Check your sizes!!!");
+
+private:
+	tl::lent_ptr<const IFilesystem> m_filesystem;
+
+	mutable std::mutex m_zipMapSourceLock;
+	tl::unique_ptr<IMapSource> m_zipMapSource;
+
+	AbsPath m_zipFileLocation;
+
+	tl::string m_encryptionKey;
+	uint32_t m_encryptionRounds = 32;
+
+	tl::vector_map<uint64_t, EntryIndex> m_pathToIndex;
+	tl::vector<File> m_files;
+	tl::vector<Folder> m_folders;
+	tl::vector<EntryIndex> m_childrenIndices; // these are indices inside the m_files or m_folders
+
+	void buildUpPath(RelPath& path, const Folder& folder, uint32_t toIndex) const;
+	void buildUpPath(RelPath& path, const File& file, uint32_t toIndex) const;
+};
+
+}

include/fs/zip/ZipReader.h

@@ -0,0 +1,63 @@
+#pragma once
+#include "ZipBase.h"
+#include "tl/memory_buffer.h"
+#include "tl/result.h"
+#include "tl/vector.h"
+
+namespace fs
+{
+
+//////////////////////////////////////////////////////////////////////////
+
+class ISource;
+
+//////////////////////////////////////////////////////////////////////////
+
+class ZipReader : public ZipBase
+{
+public:
+	enum class ErrorCode : uint8_t
+	{
+		InvalidStream,
+		BadSignature,
+		BadOffset,
+		CorruptedFile,
+	};
+	typedef tl::error<ErrorCode> Error;
+
+	typedef tl::result<ZipReader, Error> CreateResult;
+
+	static CreateResult create(ISource& source);
+
+	struct Entry
+	{
+		tl::string name;
+		CompressionMethod compressionMethod = CompressionMethod::Store;
+		GeneralBitFlags generalBitFlags;
+		uint64_t uncompressedSize = 0;
+		uint64_t compressedSize = 0;
+		uint32_t crc32 = 0;
+		uint64_t localHeaderOffset = 0;
+		uint64_t dataOffset = 0;
+		time_t lastModTimePoint = 0;
+		tl::memory_buffer comment;
+	};
+
+	size_t getEntryCount() const;
+	const Entry& getEntry(size_t index) const;
+
+	typedef tl::result<void, Error> SeekResult;
+	SeekResult seekSourceToEntryData(const Entry& entry, ISource& o_source) const;
+
+private:
+	explicit ZipReader(tl::vector<Entry> entries);
+
+	typedef tl::result<tl::pair<uint64_t, uint64_t>, Error> CentralDirectoryResult;
+	static CentralDirectoryResult findCentralDirectoryBounds(ISource& source);
+
+	tl::vector<Entry> m_entries;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+}

include/fs/zip/ZipUtils.h

@@ -0,0 +1,19 @@
+#pragma once
+#include "fs/Error.h"
+#include "fs/Api.h"
+#include "fs/IFilesystem.h"
+
+namespace fs
+{
+
+class ISink;
+
+typedef tl::result<void, Error> UnzipResult;
+FS_API UnzipResult unzipSource(IFilesystem& dstFilesystem, const AbsPath& filePath, tl::unique_ref<IMapSource> source);
+FS_API UnzipResult unzipFile(IFilesystem& dstFilesystem, const AbsPath& filePath, tl::lent_ref<const IFilesystem> filesystem, AbsPath srcFilePath);
+
+typedef tl::result<void, Error> ZipResult;
+FS_API ZipResult zipToSink(ISink& sink, tl::lent_ref<IFilesystem> filesystem, const AbsPath& path, uint8_t compressionLevel, size_t fileDataAlignment = 0);
+FS_API ZipResult zipToFile(IFilesystem& dstFilesystem, const AbsPath& dstFilePath, tl::lent_ref<IFilesystem> filesystem, const AbsPath& path, uint8_t compressionLevel, size_t fileDataAlignment = 0);
+
+}

include/fs/zip/ZipWriter.h

@@ -0,0 +1,77 @@
+#pragma once
+
+#include "ZipBase.h"
+#include "fs/Api.h"
+#include <tl/memory_buffer.h>
+#include <tl/span.h>
+#include <tl/vector.h>
+#include <tl/result.h>
+#include <tl/optional.h>
+
+namespace fs
+{
+//////////////////////////////////////////////////////////////////////////
+
+class IStreamSource;
+class IStreamSink;
+class ISink;
+
+//////////////////////////////////////////////////////////////////////////
+
+class FS_API ZipWriter : public ZipBase
+{
+public:
+	struct DataWriterPayload
+	{
+		CompressionMethod compressionMethod;
+		uint32_t uncompressedCRC32 = 0; //mandatory!!!
+		uint64_t uncompressedSize = 0; //optional, will be set to the written size if == 0
+	};
+
+	using DataWriterError = tl::generic_error;
+	using DataWriterResult = tl::result<DataWriterPayload, DataWriterError>;
+
+	using DataWriter = tl::function<DataWriterResult(IStreamSink& sink)>;
+
+	ZipWriter(ISink& sink, size_t fileDataAlignment);
+	~ZipWriter() = default;
+
+	tl::result<void> addFile(tl::string name, const DataWriter& dataWriter);
+
+	tl::result<void> addFile(tl::string name, tl::optional<time_t> lastModificationTime, const DataWriter& dataWriter);
+
+	tl::result<void> addFile(tl::string name,
+	                         tl::span<const uint8_t> comment,
+	                         tl::span<const uint8_t> extraField,
+	                         GeneralBitFlags generalBitFlags,
+	                         tl::optional<time_t> lastModificationTime,
+	                         const DataWriter& dataWriter);
+
+	void finish();
+
+private:
+	uint64_t computePadding(uint64_t offset, const LocalFileHeader& header) const;
+
+	ISink& m_sink;
+	size_t m_fileDataAlignment = 0;
+
+	struct CentralDirectoryRecord
+	{
+		uint64_t compressedSize = 0;
+		uint64_t uncompressedSize = 0;
+		uint64_t localHeaderOffset = 0;
+
+		CentralDirectoryFileHeader header;
+		tl::string name;
+		tl::memory_buffer comment;
+		tl::memory_buffer extraField;
+		bool zip64 = false;
+	};
+
+	tl::vector<CentralDirectoryRecord> m_records;
+
+	tl::memory_buffer m_buffer;
+};
+
+//////////////////////////////////////////////////////////////////////////
+}