diff --git a/.editorconfig b/.editorconfig index d05495c2..017bf61f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -6,11 +6,16 @@ indent_style = space charset = utf-8 end_of_line = lf insert_final_newline = true +trim_trailing_whitespace = true # Code files [*.{cs, csx, vb, vbx}] indent_size = 4 +# Github yaml files +[*.yml] +indent_size = 2 + # XML project files [*.{csproj, vbproj, vcxproj, vcxproj.filters, proj, projitems, shproj}] indent_size = 2 @@ -95,7 +100,6 @@ dotnet_naming_rule.instance_fields_should_be_camel_case.style = instance_field_s dotnet_naming_symbols.instance_fields.applicable_kinds = field dotnet_naming_style.instance_field_style.capitalization = camel_case -dotnet_naming_style.instance_field_style.required_prefix = _ # Locals and parameters are camelCase dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion @@ -124,26 +128,9 @@ dotnet_naming_symbols.all_members.applicable_kinds = * dotnet_naming_style.pascal_case_style.capitalization = pascal_case -# Async methods should have "Async" suffix -dotnet_naming_rule.async_methods_end_in_async.symbols = any_async_methods -dotnet_naming_rule.async_methods_end_in_async.style = end_in_async -dotnet_naming_rule.async_methods_end_in_async.severity = warning - -dotnet_naming_symbols.any_async_methods.applicable_kinds = method -dotnet_naming_symbols.any_async_methods.applicable_accessibilities = * -dotnet_naming_symbols.any_async_methods.required_modifiers = async - -dotnet_naming_style.end_in_async.required_prefix = -dotnet_naming_style.end_in_async.required_suffix = Async -dotnet_naming_style.end_in_async.capitalization = pascal_case -dotnet_naming_style.end_in_async.word_separator = - # error RS2008: Enable analyzer release tracking for the analyzer project containing rule '{0}' dotnet_diagnostic.RS2008.severity = none -# IDE0005: Remove unnecessary import -dotnet_diagnostic.IDE0005.severity = warning - # IDE0007: Use `var` instead of explicit type dotnet_diagnostic.IDE0007.severity = warning @@ -161,6 +148,11 @@ dotnet_diagnostic.IDE0044.severity = warning # CSharp code style settings: [*.cs] + +# Require file header OR A source file contains a header that does not match the required text +file_header_template = This file is subject to the terms and conditions defined\nin file 'LICENSE', which is part of this source code package. +dotnet_diagnostic.IDE0073.severity = error + # Newline settings csharp_new_line_before_open_brace = all csharp_new_line_before_else = true diff --git a/.gitattributes b/.gitattributes index a6e03fce..ff739929 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,4 @@ *.cs text eol=lf -*.csproj text eol=crlf +*.csproj text eol=lf *.config eol=lf *.json eol=lf diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index d9e2ae9c..941391c0 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -32,17 +32,24 @@ body: validations: required: true - type: input - id: dotnet-version + id: version attributes: label: Version - description: What version of DepotDownloader are you running on? + description: What version of DepotDownloader are using? + validations: + required: true + - type: input + id: command + attributes: + label: Command + description: Specify the full command you used (except for username and password) validations: required: true - type: textarea id: logs attributes: label: Relevant log output - description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. Run with `-debug` parameter to get additional output. render: shell - type: textarea id: additional-info diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c0725e41..686a2cb5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,7 +33,7 @@ jobs: dotnet-version: 8.0.x - name: Build - run: dotnet publish DepotDownloader/DepotDownloader.csproj -c ${{ matrix.configuration }} -o artifacts + run: dotnet publish DepotDownloader/DepotDownloader.csproj -c ${{ matrix.configuration }} -o artifacts /p:ContinuousIntegrationBuild=true - name: Upload artifact uses: actions/upload-artifact@v4 @@ -126,3 +126,47 @@ jobs: name: DepotDownloader-macos-arm64 path: selfcontained-osx-arm64 if-no-files-found: error + + release: + if: startsWith(github.ref, 'refs/tags/') + needs: build + runs-on: ubuntu-latest + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Display artifacts folder structure + run: ls -Rl + working-directory: artifacts + + - name: Create release files + run: | + set -eux + mkdir release + chmod +x artifacts/DepotDownloader-linux-x64/DepotDownloader + chmod +x artifacts/DepotDownloader-linux-arm/DepotDownloader + chmod +x artifacts/DepotDownloader-linux-arm64/DepotDownloader + chmod +x artifacts/DepotDownloader-macos-x64/DepotDownloader + chmod +x artifacts/DepotDownloader-macos-arm64/DepotDownloader + zip -9j release/DepotDownloader-framework.zip artifacts/DepotDownloader-framework/* + zip -9j release/DepotDownloader-windows-x64.zip artifacts/DepotDownloader-windows-x64/* + zip -9j release/DepotDownloader-windows-arm64.zip artifacts/DepotDownloader-windows-arm64/* + zip -9j release/DepotDownloader-linux-x64.zip artifacts/DepotDownloader-linux-x64/* + zip -9j release/DepotDownloader-linux-arm.zip artifacts/DepotDownloader-linux-arm/* + zip -9j release/DepotDownloader-linux-arm64.zip artifacts/DepotDownloader-linux-arm64/* + zip -9j release/DepotDownloader-macos-x64.zip artifacts/DepotDownloader-macos-x64/* + zip -9j release/DepotDownloader-macos-arm64.zip artifacts/DepotDownloader-macos-arm64/* + + - name: Display structure of archived files + run: ls -Rl + working-directory: release + + - name: Release + uses: softprops/action-gh-release@v2 + with: + draft: true + files: release/* + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sk2-ci.yml b/.github/workflows/sk2-ci.yml index 8827a46f..b7463d9c 100644 --- a/.github/workflows/sk2-ci.yml +++ b/.github/workflows/sk2-ci.yml @@ -30,12 +30,12 @@ jobs: dotnet add DepotDownloader/DepotDownloader.csproj package SteamKit2 --prerelease - name: Build - run: dotnet publish DepotDownloader/DepotDownloader.csproj -c ${{ matrix.configuration }} -o artifacts + run: dotnet publish DepotDownloader/DepotDownloader.csproj -c ${{ matrix.configuration }} -o artifacts /p:ContinuousIntegrationBuild=true - name: Upload artifact uses: actions/upload-artifact@v4 if: matrix.configuration == 'Release' with: - name: DepotDownloader-${{ runner.os }} + name: DepotDownloader-${{ matrix.runs-on }} path: artifacts if-no-files-found: error diff --git a/.github/workflows/winget.yml b/.github/workflows/winget.yml new file mode 100644 index 00000000..0241c1cc --- /dev/null +++ b/.github/workflows/winget.yml @@ -0,0 +1,29 @@ +name: WinGet submission on release + +on: + workflow_dispatch: + release: + types: [published] + +jobs: + winget: + name: Publish winget package + runs-on: windows-latest + steps: + - name: Submit package to Windows Package Manager Community Repository + run: | + $wingetPackage = "SteamRE.DepotDownloader" + + $github = Invoke-RestMethod -uri "https://api.github.com/repos/SteamRE/DepotDownloader/releases" + + $targetRelease = $github | Where-Object -Property name -match '^DepotDownloader' | Select -First 1 + $assets = $targetRelease | Select -ExpandProperty assets -First 1 + $zipX64Url = $assets | Where-Object -Property name -match 'DepotDownloader-windows-x64.zip' | Select -ExpandProperty browser_download_url + $zipArm64Url = $assets | Where-Object -Property name -match 'DepotDownloader-windows-arm64.zip' | Select -ExpandProperty browser_download_url + $ver = $targetRelease.tag_name -ireplace '^(DepotDownloader[ _])?v?' + + # getting latest wingetcreate file + iwr https://aka.ms/wingetcreate/latest -OutFile wingetcreate.exe + + # how to create a token: https://github.com/microsoft/winget-create?tab=readme-ov-file#github-personal-access-token-classic-permissions + .\wingetcreate.exe update $wingetPackage --submit --version $ver --urls "$zipX64Url" "$zipArm64Url" --token "${{ secrets.PT_WINGET }}" diff --git a/DepotDownloader/AccountSettingsStore.cs b/DepotDownloader/AccountSettingsStore.cs index 2374a92a..7e1594be 100644 --- a/DepotDownloader/AccountSettingsStore.cs +++ b/DepotDownloader/AccountSettingsStore.cs @@ -1,3 +1,6 @@ +// This file is subject to the terms and conditions defined +// in file 'LICENSE', which is part of this source code package. + using System; using System.Collections.Concurrent; using System.Collections.Generic; diff --git a/DepotDownloader/Ansi.cs b/DepotDownloader/Ansi.cs new file mode 100644 index 00000000..b17b5972 --- /dev/null +++ b/DepotDownloader/Ansi.cs @@ -0,0 +1,54 @@ +// This file is subject to the terms and conditions defined +// in file 'LICENSE', which is part of this source code package. + +using System; +using Spectre.Console; + +namespace DepotDownloader; + +static class Ansi +{ + // https://conemu.github.io/en/AnsiEscapeCodes.html#ConEmu_specific_OSC + // https://learn.microsoft.com/en-us/windows/terminal/tutorials/progress-bar-sequences + public enum ProgressState + { + Hidden = 0, + Default = 1, + Error = 2, + Indeterminate = 3, + Warning = 4, + } + + const char ESC = (char)0x1B; + const char BEL = (char)0x07; + + private static bool useProgress; + + public static void Init() + { + if (Console.IsInputRedirected || Console.IsOutputRedirected) + { + return; + } + + var (supportsAnsi, legacyConsole) = AnsiDetector.Detect(stdError: false, upgrade: true); + + useProgress = supportsAnsi && !legacyConsole; + } + + public static void Progress(ulong downloaded, ulong total) + { + var progress = (byte)MathF.Round(downloaded / (float)total * 100.0f); + Progress(ProgressState.Default, progress); + } + + public static void Progress(ProgressState state, byte progress = 0) + { + if (!useProgress) + { + return; + } + + Console.Write($"{ESC}]9;4;{(byte)state};{progress}{BEL}"); + } +} diff --git a/DepotDownloader/AnsiDetector.cs b/DepotDownloader/AnsiDetector.cs new file mode 100644 index 00000000..2110d9cc --- /dev/null +++ b/DepotDownloader/AnsiDetector.cs @@ -0,0 +1,134 @@ +// Copied from https://github.com/spectreconsole/spectre.console/blob/d79e6adc5f8e637fb35c88f987023ffda6707243/src/Spectre.Console/Internal/Backends/Ansi/AnsiDetector.cs +// MIT License - Copyright(c) 2020 Patrik Svensson, Phil Scott, Nils Andresen +// which is partially based on https://github.com/keqingrong/supports-ansi/blob/master/index.js +// + +using System; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text.RegularExpressions; + +namespace Spectre.Console; + +internal static class AnsiDetector +{ + private static readonly Regex[] _regexes = + [ + new("^xterm"), // xterm, PuTTY, Mintty + new("^rxvt"), // RXVT + new("^eterm"), // Eterm + new("^screen"), // GNU screen, tmux + new("tmux"), // tmux + new("^vt100"), // DEC VT series + new("^vt102"), // DEC VT series + new("^vt220"), // DEC VT series + new("^vt320"), // DEC VT series + new("ansi"), // ANSI + new("scoansi"), // SCO ANSI + new("cygwin"), // Cygwin, MinGW + new("linux"), // Linux console + new("konsole"), // Konsole + new("bvterm"), // Bitvise SSH Client + new("^st-256color"), // Suckless Simple Terminal, st + new("alacritty"), // Alacritty + ]; + + public static (bool SupportsAnsi, bool LegacyConsole) Detect(bool stdError, bool upgrade) + { + // Running on Windows? + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + // Running under ConEmu? + var conEmu = Environment.GetEnvironmentVariable("ConEmuANSI"); + if (!string.IsNullOrEmpty(conEmu) && conEmu.Equals("On", StringComparison.OrdinalIgnoreCase)) + { + return (true, false); + } + + var supportsAnsi = Windows.SupportsAnsi(upgrade, stdError, out var legacyConsole); + return (supportsAnsi, legacyConsole); + } + + return DetectFromTerm(); + } + + private static (bool SupportsAnsi, bool LegacyConsole) DetectFromTerm() + { + // Check if the terminal is of type ANSI/VT100/xterm compatible. + var term = Environment.GetEnvironmentVariable("TERM"); + if (!string.IsNullOrWhiteSpace(term)) + { + if (_regexes.Any(regex => regex.IsMatch(term))) + { + return (true, false); + } + } + + return (false, true); + } + + private static class Windows + { + private const int STD_OUTPUT_HANDLE = -11; + private const int STD_ERROR_HANDLE = -12; + private const uint ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004; + private const uint DISABLE_NEWLINE_AUTO_RETURN = 0x0008; + + [DllImport("kernel32.dll")] + private static extern bool GetConsoleMode(IntPtr hConsoleHandle, out uint lpMode); + + [DllImport("kernel32.dll")] + private static extern bool SetConsoleMode(IntPtr hConsoleHandle, uint dwMode); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern IntPtr GetStdHandle(int nStdHandle); + + [DllImport("kernel32.dll")] + public static extern uint GetLastError(); + + public static bool SupportsAnsi(bool upgrade, bool stdError, out bool isLegacy) + { + isLegacy = false; + + try + { + var @out = GetStdHandle(stdError ? STD_ERROR_HANDLE : STD_OUTPUT_HANDLE); + if (!GetConsoleMode(@out, out var mode)) + { + // Could not get console mode, try TERM (set in cygwin, WSL-Shell). + var (ansiFromTerm, legacyFromTerm) = DetectFromTerm(); + + isLegacy = ansiFromTerm ? legacyFromTerm : isLegacy; + return ansiFromTerm; + } + + if ((mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING) == 0) + { + isLegacy = true; + + if (!upgrade) + { + return false; + } + + // Try enable ANSI support. + mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN; + if (!SetConsoleMode(@out, mode)) + { + // Enabling failed. + return false; + } + + isLegacy = false; + } + + return true; + } + catch + { + // All we know here is that we don't support ANSI. + return false; + } + } + } +} diff --git a/DepotDownloader/CDNClientPool.cs b/DepotDownloader/CDNClientPool.cs index 2ed0a086..596beea6 100644 --- a/DepotDownloader/CDNClientPool.cs +++ b/DepotDownloader/CDNClientPool.cs @@ -1,3 +1,6 @@ +// This file is subject to the terms and conditions defined +// in file 'LICENSE', which is part of this source code package. + using System; using System.Collections.Concurrent; using System.Collections.Generic; diff --git a/DepotDownloader/ContentDownloader.cs b/DepotDownloader/ContentDownloader.cs index 83d902f8..9fd9fcd4 100644 --- a/DepotDownloader/ContentDownloader.cs +++ b/DepotDownloader/ContentDownloader.cs @@ -1,4 +1,8 @@ +// This file is subject to the terms and conditions defined +// in file 'LICENSE', which is part of this source code package. + using System; +using System.Buffers; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; @@ -110,7 +114,7 @@ static bool AccountHasAccess(uint depotId) IEnumerable licenseQuery; if (steam3.steamUser.SteamID.AccountType == EAccountType.AnonUser) { - licenseQuery = new List { 17906 }; + licenseQuery = [17906]; } else { @@ -244,7 +248,7 @@ static ulong GetSteam3DepotManifest(uint depotId, uint appId, string branch) byte[] manifest_bytes; try { - manifest_bytes = CryptoHelper.SymmetricDecryptECB(input, appBetaPassword); + manifest_bytes = Util.SymmetricDecryptECB(input, appBetaPassword); } catch (Exception e) { @@ -611,20 +615,23 @@ private class FileStreamData private class GlobalDownloadCounter { - public ulong TotalBytesCompressed; - public ulong TotalBytesUncompressed; + public ulong completeDownloadSize; + public ulong totalBytesCompressed; + public ulong totalBytesUncompressed; } private class DepotDownloadCounter { - public ulong CompleteDownloadSize; - public ulong SizeDownloaded; - public ulong DepotBytesCompressed; - public ulong DepotBytesUncompressed; + public ulong completeDownloadSize; + public ulong sizeDownloaded; + public ulong depotBytesCompressed; + public ulong depotBytesUncompressed; } private static async Task DownloadSteam3Async(List depots) { + Ansi.Progress(Ansi.ProgressState.Indeterminate); + var cts = new CancellationTokenSource(); cdnPool.ExhaustedToken = cts; @@ -635,7 +642,7 @@ private static async Task DownloadSteam3Async(List depots) // First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup foreach (var depot in depots) { - var depotFileData = await ProcessDepotManifestAndFiles(cts, depot); + var depotFileData = await ProcessDepotManifestAndFiles(cts, depot, downloadCounter); if (depotFileData != null) { @@ -666,11 +673,13 @@ private static async Task DownloadSteam3Async(List depots) await DownloadSteam3AsyncDepotFiles(cts, downloadCounter, depotFileData, allFileNamesAllDepots); } + Ansi.Progress(Ansi.ProgressState.Hidden); + Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", - downloadCounter.TotalBytesCompressed, downloadCounter.TotalBytesUncompressed, depots.Count); + downloadCounter.totalBytesCompressed, downloadCounter.totalBytesUncompressed, depots.Count); } - private static async Task ProcessDepotManifestAndFiles(CancellationTokenSource cts, DepotDownloadInfo depot) + private static async Task ProcessDepotManifestAndFiles(CancellationTokenSource cts, DepotDownloadInfo depot, GlobalDownloadCounter downloadCounter) { var depotCounter = new DepotDownloadCounter(); @@ -752,7 +761,7 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat } else { - Console.Write("Downloading depot manifest..."); + Console.Write("Downloading depot manifest... "); DepotManifest depotManifest = null; ulong manifestRequestCode = 0; @@ -768,6 +777,13 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat { connection = cdnPool.GetConnection(cts.Token); + string cdnToken = null; + if (steam3.CDNAuthTokens.TryGetValue((depot.DepotId, connection.Host), out var authTokenCallbackPromise)) + { + var result = await authTokenCallbackPromise.Task; + cdnToken = result.Token; + } + var now = DateTime.Now; // In order to download this manifest, we need the current manifest request code @@ -801,7 +817,8 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat manifestRequestCode, connection, depot.DepotKey, - cdnPool.ProxyServer).ConfigureAwait(false); + cdnPool.ProxyServer, + cdnToken).ConfigureAwait(false); cdnPool.ReturnConnection(connection); } @@ -811,11 +828,21 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat } catch (SteamKitWebRequestException e) { + // If the CDN returned 403, attempt to get a cdn auth if we didn't yet + if (e.StatusCode == HttpStatusCode.Forbidden && !steam3.CDNAuthTokens.ContainsKey((depot.DepotId, connection.Host))) + { + await steam3.RequestCDNAuthToken(depot.AppId, depot.DepotId, connection); + + cdnPool.ReturnConnection(connection); + + continue; + } + cdnPool.ReturnBrokenConnection(connection); if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) { - Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.DepotId, depot.ManifestId); + Console.WriteLine("Encountered {2} for depot manifest {0} {1}. Aborting.", depot.DepotId, depot.ManifestId, (int)e.StatusCode); break; } @@ -890,7 +917,8 @@ private static async Task ProcessDepotManifestAndFiles(Cancellat Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); - depotCounter.CompleteDownloadSize += file.TotalSize; + downloadCounter.completeDownloadSize += file.TotalSize; + depotCounter.completeDownloadSize += file.TotalSize; } }); @@ -919,7 +947,7 @@ private static async Task DownloadSteam3AsyncDepotFiles(CancellationTokenSource await Util.InvokeAsync( files.Select(file => new Func(async () => - await Task.Run(() => DownloadSteam3AsyncDepotFile(cts, depotFilesData, file, networkChunkQueue)))), + await Task.Run(() => DownloadSteam3AsyncDepotFile(cts, downloadCounter, depotFilesData, file, networkChunkQueue)))), maxDegreeOfParallelism: Config.MaxDownloads ); @@ -962,11 +990,12 @@ await Task.Run(() => DownloadSteam3AsyncDepotFileChunk(cts, downloadCounter, dep DepotConfigStore.Instance.InstalledManifestIDs[depot.DepotId] = depot.ManifestId; DepotConfigStore.Save(); - Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.DepotId, depotCounter.DepotBytesCompressed, depotCounter.DepotBytesUncompressed); + Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.DepotId, depotCounter.depotBytesCompressed, depotCounter.depotBytesUncompressed); } private static void DownloadSteam3AsyncDepotFile( CancellationTokenSource cts, + GlobalDownloadCounter downloadCounter, DepotFilesData depotFilesData, ProtoManifest.FileData file, ConcurrentQueue<(FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData)> networkChunkQueue) @@ -1053,10 +1082,7 @@ private static void DownloadSteam3AsyncDepotFile( { fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin); - var tmp = new byte[match.OldChunk.UncompressedLength]; - fsOld.Read(tmp, 0, tmp.Length); - - var adler = Util.AdlerHash(tmp); + var adler = Util.AdlerHash(fsOld, (int)match.OldChunk.UncompressedLength); if (!adler.SequenceEqual(match.OldChunk.Checksum)) { neededChunks.Add(match.NewChunk); @@ -1125,8 +1151,13 @@ private static void DownloadSteam3AsyncDepotFile( { lock (depotDownloadCounter) { - depotDownloadCounter.SizeDownloaded += file.TotalSize; - Console.WriteLine("{0,6:#00.00}% {1}", (depotDownloadCounter.SizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath); + depotDownloadCounter.sizeDownloaded += file.TotalSize; + Console.WriteLine("{0,6:#00.00}% {1}", (depotDownloadCounter.sizeDownloaded / (float)depotDownloadCounter.completeDownloadSize) * 100.0f, fileFinalPath); + } + + lock (downloadCounter) + { + downloadCounter.completeDownloadSize -= file.TotalSize; } return; @@ -1135,7 +1166,12 @@ private static void DownloadSteam3AsyncDepotFile( var sizeOnDisk = (file.TotalSize - (ulong)neededChunks.Select(x => (long)x.UncompressedLength).Sum()); lock (depotDownloadCounter) { - depotDownloadCounter.SizeDownloaded += sizeOnDisk; + depotDownloadCounter.sizeDownloaded += sizeOnDisk; + } + + lock (downloadCounter) + { + downloadCounter.completeDownloadSize -= sizeOnDisk; } } @@ -1175,91 +1211,122 @@ private static async Task DownloadSteam3AsyncDepotFileChunk( var depot = depotFilesData.depotDownloadInfo; var depotDownloadCounter = depotFilesData.depotCounter; - var chunkID = Util.EncodeHexString(chunk.ChunkID); + var chunkID = Convert.ToHexString(chunk.ChunkID).ToLowerInvariant(); var data = new DepotManifest.ChunkData { ChunkID = chunk.ChunkID, - Checksum = chunk.Checksum, + Checksum = BitConverter.ToUInt32(chunk.Checksum), Offset = chunk.Offset, CompressedLength = chunk.CompressedLength, UncompressedLength = chunk.UncompressedLength }; - DepotChunk chunkData = null; + var written = 0; + var chunkBuffer = ArrayPool.Shared.Rent((int)data.UncompressedLength); - do + try { - cts.Token.ThrowIfCancellationRequested(); + do + { + cts.Token.ThrowIfCancellationRequested(); - Server connection = null; + Server connection = null; - try - { - connection = cdnPool.GetConnection(cts.Token); + try + { + connection = cdnPool.GetConnection(cts.Token); - DebugLog.WriteLine("ContentDownloader", "Downloading chunk {0} from {1} with {2}", chunkID, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy"); - chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync( - depot.DepotId, - data, - connection, - depot.DepotKey, - cdnPool.ProxyServer).ConfigureAwait(false); + string cdnToken = null; + if (steam3.CDNAuthTokens.TryGetValue((depot.DepotId, connection.Host), out var authTokenCallbackPromise)) + { + var result = await authTokenCallbackPromise.Task; + cdnToken = result.Token; + } - cdnPool.ReturnConnection(connection); - } - catch (TaskCanceledException) - { - Console.WriteLine("Connection timeout downloading chunk {0}", chunkID); - } - catch (SteamKitWebRequestException e) - { - cdnPool.ReturnBrokenConnection(connection); + DebugLog.WriteLine("ContentDownloader", "Downloading chunk {0} from {1} with {2}", chunkID, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy"); + written = await cdnPool.CDNClient.DownloadDepotChunkAsync( + depot.DepotId, + data, + connection, + chunkBuffer, + depot.DepotKey, + cdnPool.ProxyServer, + cdnToken).ConfigureAwait(false); + + cdnPool.ReturnConnection(connection); + + break; + } + catch (TaskCanceledException) + { + Console.WriteLine("Connection timeout downloading chunk {0}", chunkID); + } + catch (SteamKitWebRequestException e) + { + // If the CDN returned 403, attempt to get a cdn auth if we didn't yet, + // if auth task already exists, make sure it didn't complete yet, so that it gets awaited above + if (e.StatusCode == HttpStatusCode.Forbidden && + (!steam3.CDNAuthTokens.TryGetValue((depot.DepotId, connection.Host), out var authTokenCallbackPromise) || !authTokenCallbackPromise.Task.IsCompleted)) + { + await steam3.RequestCDNAuthToken(depot.AppId, depot.DepotId, connection); + + cdnPool.ReturnConnection(connection); + + continue; + } + + cdnPool.ReturnBrokenConnection(connection); - if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) + if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) + { + Console.WriteLine("Encountered {1} for chunk {0}. Aborting.", chunkID, (int)e.StatusCode); + break; + } + + Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode); + } + catch (OperationCanceledException) { - Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID); break; } + catch (Exception e) + { + cdnPool.ReturnBrokenConnection(connection); + Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message); + } + } while (written == 0); - Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode); - } - catch (OperationCanceledException) + if (written == 0) { - break; + Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.DepotId); + cts.Cancel(); } - catch (Exception e) - { - cdnPool.ReturnBrokenConnection(connection); - Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message); - } - } while (chunkData == null); - if (chunkData == null) - { - Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.DepotId); - cts.Cancel(); - } + // Throw the cancellation exception if requested so that this task is marked failed + cts.Token.ThrowIfCancellationRequested(); - // Throw the cancellation exception if requested so that this task is marked failed - cts.Token.ThrowIfCancellationRequested(); + try + { + await fileStreamData.fileLock.WaitAsync().ConfigureAwait(false); - try - { - await fileStreamData.fileLock.WaitAsync().ConfigureAwait(false); + if (fileStreamData.fileStream == null) + { + var fileFinalPath = Path.Combine(depot.InstallDir, file.FileName); + fileStreamData.fileStream = File.Open(fileFinalPath, FileMode.Open); + } - if (fileStreamData.fileStream == null) + fileStreamData.fileStream.Seek((long)data.Offset, SeekOrigin.Begin); + await fileStreamData.fileStream.WriteAsync(chunkBuffer.AsMemory(0, written), cts.Token); + } + finally { - var fileFinalPath = Path.Combine(depot.InstallDir, file.FileName); - fileStreamData.fileStream = File.Open(fileFinalPath, FileMode.Open); + fileStreamData.fileLock.Release(); } - - fileStreamData.fileStream.Seek((long)chunkData.ChunkInfo.Offset, SeekOrigin.Begin); - await fileStreamData.fileStream.WriteAsync(chunkData.Data.AsMemory(0, chunkData.Data.Length), cts.Token); } finally { - fileStreamData.fileLock.Release(); + ArrayPool.Shared.Return(chunkBuffer); } var remainingChunks = Interlocked.Decrement(ref fileStreamData.chunksToDownload); @@ -1272,22 +1339,24 @@ private static async Task DownloadSteam3AsyncDepotFileChunk( ulong sizeDownloaded = 0; lock (depotDownloadCounter) { - sizeDownloaded = depotDownloadCounter.SizeDownloaded + (ulong)chunkData.Data.Length; - depotDownloadCounter.SizeDownloaded = sizeDownloaded; - depotDownloadCounter.DepotBytesCompressed += chunk.CompressedLength; - depotDownloadCounter.DepotBytesUncompressed += chunk.UncompressedLength; + sizeDownloaded = depotDownloadCounter.sizeDownloaded + (ulong)written; + depotDownloadCounter.sizeDownloaded = sizeDownloaded; + depotDownloadCounter.depotBytesCompressed += chunk.CompressedLength; + depotDownloadCounter.depotBytesUncompressed += chunk.UncompressedLength; } lock (downloadCounter) { - downloadCounter.TotalBytesCompressed += chunk.CompressedLength; - downloadCounter.TotalBytesUncompressed += chunk.UncompressedLength; + downloadCounter.totalBytesCompressed += chunk.CompressedLength; + downloadCounter.totalBytesUncompressed += chunk.UncompressedLength; + + Ansi.Progress(downloadCounter.totalBytesUncompressed, downloadCounter.completeDownloadSize); } if (remainingChunks == 0) { var fileFinalPath = Path.Combine(depot.InstallDir, file.FileName); - Console.WriteLine("{0,6:#00.00}% {1}", (sizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath); + Console.WriteLine("{0,6:#00.00}% {1}", (sizeDownloaded / (float)depotDownloadCounter.completeDownloadSize) * 100.0f, fileFinalPath); } } diff --git a/DepotDownloader/DepotConfigStore.cs b/DepotDownloader/DepotConfigStore.cs index 99f052c5..15e15393 100644 --- a/DepotDownloader/DepotConfigStore.cs +++ b/DepotDownloader/DepotConfigStore.cs @@ -1,3 +1,6 @@ +// This file is subject to the terms and conditions defined +// in file 'LICENSE', which is part of this source code package. + using System; using System.Collections.Generic; using System.IO; diff --git a/DepotDownloader/DepotDownloader.csproj b/DepotDownloader/DepotDownloader.csproj index 47cfd284..16a814ea 100644 --- a/DepotDownloader/DepotDownloader.csproj +++ b/DepotDownloader/DepotDownloader.csproj @@ -4,17 +4,28 @@ net8.0 true LatestMajor - 2.6.0 + 2.7.1 Steam Downloading Utility SteamRE Team Copyright © SteamRE Team 2024 ..\Icon\DepotDownloader.ico - true + true + true + + Always + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + - - + + diff --git a/DepotDownloader/DownloadConfig.cs b/DepotDownloader/DownloadConfig.cs index 25bf5e70..c0d4f6cb 100644 --- a/DepotDownloader/DownloadConfig.cs +++ b/DepotDownloader/DownloadConfig.cs @@ -1,4 +1,7 @@ -using System.Collections.Generic; +// This file is subject to the terms and conditions defined +// in file 'LICENSE', which is part of this source code package. + +using System.Collections.Generic; using System.Text.RegularExpressions; namespace DepotDownloader diff --git a/DepotDownloader/HttpClientFactory.cs b/DepotDownloader/HttpClientFactory.cs index aa642bc7..7a622117 100644 --- a/DepotDownloader/HttpClientFactory.cs +++ b/DepotDownloader/HttpClientFactory.cs @@ -1,3 +1,6 @@ +// This file is subject to the terms and conditions defined +// in file 'LICENSE', which is part of this source code package. + using System.IO; using System.Net.Http; using System.Net.Http.Headers; diff --git a/DepotDownloader/HttpDiagnosticEventListener.cs b/DepotDownloader/HttpDiagnosticEventListener.cs index f3b11429..26026561 100644 --- a/DepotDownloader/HttpDiagnosticEventListener.cs +++ b/DepotDownloader/HttpDiagnosticEventListener.cs @@ -1,3 +1,6 @@ +// This file is subject to the terms and conditions defined +// in file 'LICENSE', which is part of this source code package. + using System; using System.Diagnostics.Tracing; using System.Text; diff --git a/DepotDownloader/NativeMethods.txt b/DepotDownloader/NativeMethods.txt new file mode 100644 index 00000000..8f2bb821 --- /dev/null +++ b/DepotDownloader/NativeMethods.txt @@ -0,0 +1,2 @@ +GetConsoleProcessList +MessageBox diff --git a/DepotDownloader/PlatformUtilities.cs b/DepotDownloader/PlatformUtilities.cs index 9ffb70fb..a4f8fd77 100644 --- a/DepotDownloader/PlatformUtilities.cs +++ b/DepotDownloader/PlatformUtilities.cs @@ -1,126 +1,52 @@ +// This file is subject to the terms and conditions defined +// in file 'LICENSE', which is part of this source code package. + using System; +using System.IO; using System.Runtime.InteropServices; +using System.Runtime.Versioning; namespace DepotDownloader { static class PlatformUtilities { - private const int ModeExecuteOwner = 0x0040; - private const int ModeExecuteGroup = 0x0008; - private const int ModeExecuteOther = 0x0001; - private const int ModeExecute = ModeExecuteOwner | ModeExecuteGroup | ModeExecuteOther; - - [StructLayout(LayoutKind.Explicit, Size = 144)] - private readonly struct StatLinuxX64 - { - [FieldOffset(24)] public readonly uint st_mode; - } - - [StructLayout(LayoutKind.Explicit, Size = 104)] - private readonly struct StatLinuxArm32 - { - [FieldOffset(16)] public readonly uint st_mode; - } - - [StructLayout(LayoutKind.Explicit, Size = 128)] - private readonly struct StatLinuxArm64 - { - [FieldOffset(16)] public readonly uint st_mode; - } - - [StructLayout(LayoutKind.Explicit, Size = 144)] - private readonly struct StatOSX - { - [FieldOffset(4)] public readonly ushort st_mode; - } - - [DllImport("libc", EntryPoint = "__xstat", SetLastError = true)] - private static extern int statLinuxX64(int version, string path, out StatLinuxX64 statLinux); - - [DllImport("libc", EntryPoint = "__xstat", SetLastError = true)] - private static extern int statLinuxArm32(int version, string path, out StatLinuxArm32 statLinux); - - [DllImport("libc", EntryPoint = "__xstat", SetLastError = true)] - private static extern int statLinuxArm64(int version, string path, out StatLinuxArm64 statLinux); - - [DllImport("libc", EntryPoint = "stat", SetLastError = true)] - private static extern int statOSX(string path, out StatOSX stat); - - [DllImport("libc", EntryPoint = "stat$INODE64", SetLastError = true)] - private static extern int statOSXCompat(string path, out StatOSX stat); - - [DllImport("libc", SetLastError = true)] - private static extern int chmod(string path, uint mode); - - [DllImport("libc", CallingConvention = CallingConvention.Cdecl, SetLastError = true)] - private static extern IntPtr strerror(int errno); - - private static void ThrowIf(int i) + public static void SetExecutable(string path, bool value) { - if (i == -1) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - var errno = Marshal.GetLastWin32Error(); - throw new Exception(Marshal.PtrToStringAnsi(strerror(errno))); + return; } - } - private static uint GetFileMode(string path) - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - { - switch (RuntimeInformation.ProcessArchitecture) - { - case Architecture.X64: - { - ThrowIf(statLinuxX64(1, path, out var stat)); - return stat.st_mode; - } - case Architecture.Arm: - { - ThrowIf(statLinuxArm32(3, path, out var stat)); - return stat.st_mode; - } - case Architecture.Arm64: - { - ThrowIf(statLinuxArm64(0, path, out var stat)); - return stat.st_mode; - } - } - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + const UnixFileMode ModeExecute = UnixFileMode.UserExecute | UnixFileMode.GroupExecute | UnixFileMode.OtherExecute; + + var mode = File.GetUnixFileMode(path); + var hasExecuteMask = (mode & ModeExecute) == ModeExecute; + if (hasExecuteMask != value) { - switch (RuntimeInformation.ProcessArchitecture) - { - case Architecture.X64: - { - ThrowIf(statOSXCompat(path, out var stat)); - return stat.st_mode; - } - case Architecture.Arm64: - { - ThrowIf(statOSX(path, out var stat)); - return stat.st_mode; - } - } + File.SetUnixFileMode(path, value + ? mode | ModeExecute + : mode & ~ModeExecute); } - throw new PlatformNotSupportedException(); } - public static void SetExecutable(string path, bool value) + [SupportedOSPlatform("windows5.0")] + public static void VerifyConsoleLaunch() { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + // Reference: https://devblogs.microsoft.com/oldnewthing/20160125-00/?p=92922 + var processList = new uint[2]; + var processCount = Windows.Win32.PInvoke.GetConsoleProcessList(processList); + + if (processCount != 1) { return; } - var mode = GetFileMode(path); - var hasExecuteMask = (mode & ModeExecute) == ModeExecute; - if (hasExecuteMask != value) - { - ThrowIf(chmod(path, (uint)(value - ? mode | ModeExecute - : mode & ~ModeExecute))); - } + _ = Windows.Win32.PInvoke.MessageBox( + Windows.Win32.Foundation.HWND.Null, + "Depot Downloader is a console application; there is no GUI.\n\nIf you do not pass any command line parameters, it prints usage info and exits.\n\nYou must use this from a terminal/console.", + "Depot Downloader", + Windows.Win32.UI.WindowsAndMessaging.MESSAGEBOX_STYLE.MB_OK | Windows.Win32.UI.WindowsAndMessaging.MESSAGEBOX_STYLE.MB_ICONWARNING + ); } } } diff --git a/DepotDownloader/Program.cs b/DepotDownloader/Program.cs index 8b5839f7..569c4893 100644 --- a/DepotDownloader/Program.cs +++ b/DepotDownloader/Program.cs @@ -1,3 +1,6 @@ +// This file is subject to the terms and conditions defined +// in file 'LICENSE', which is part of this source code package. + using System; using System.Collections.Generic; using System.ComponentModel; @@ -13,27 +16,40 @@ namespace DepotDownloader { class Program { - static int Main(string[] args) - => MainAsync(args).GetAwaiter().GetResult(); - - internal static readonly char[] newLineCharacters = ['\n', '\r']; - - static async Task MainAsync(string[] args) + static async Task Main(string[] args) { if (args.Length == 0) { + PrintVersion(); PrintUsage(); - return 1; + + if (OperatingSystem.IsWindowsVersionAtLeast(5, 0)) + { + PlatformUtilities.VerifyConsoleLaunch(); + } + + return 0; } + Ansi.Init(); + DebugLog.Enabled = false; AccountSettingsStore.LoadFromFile("account.config"); #region Common Options + // Not using HasParameter because it is case insensitive + if (args.Length == 1 && (args[0] == "-V" || args[0] == "--version")) + { + PrintVersion(true); + return 0; + } + if (HasParameter(args, "-debug")) { + PrintVersion(true); + DebugLog.Enabled = true; DebugLog.AddListener((category, message) => { @@ -41,9 +57,6 @@ static async Task MainAsync(string[] args) }); var httpEventListener = new HttpDiagnosticEventListener(); - - DebugLog.WriteLine("DepotDownloader", "Version: {0}", Assembly.GetExecutingAssembly().GetName().Version); - DebugLog.WriteLine("DepotDownloader", "Runtime: {0}", RuntimeInformation.FrameworkDescription); } var username = GetParameter(args, "-username") ?? GetParameter(args, "-user"); @@ -69,15 +82,19 @@ static async Task MainAsync(string[] args) try { - var fileListData = await File.ReadAllTextAsync(fileList); - var files = fileListData.Split(newLineCharacters, StringSplitOptions.RemoveEmptyEntries); - ContentDownloader.Config.UsingFileList = true; ContentDownloader.Config.FilesToDownload = new HashSet(StringComparer.OrdinalIgnoreCase); ContentDownloader.Config.FilesToDownloadRegex = []; + var files = await File.ReadAllLinesAsync(fileList); + foreach (var fileEntry in files) { + if (string.IsNullOrWhiteSpace(fileEntry)) + { + continue; + } + if (fileEntry.StartsWith(RegexPrefix)) { var rgx = new Regex(fileEntry[RegexPrefix.Length..], RegexOptions.Compiled | RegexOptions.IgnoreCase); @@ -375,46 +392,60 @@ static List GetParameterList(string[] args, string param) static void PrintUsage() { + // Do not use tabs to align parameters here because tab size may differ Console.WriteLine(); - Console.WriteLine("Usage - downloading one or all depots for an app:"); - Console.WriteLine("\tdepotdownloader -app [-depot [-manifest ]]"); - Console.WriteLine("\t\t[-username [-password ]] [other options]"); + Console.WriteLine("Usage: downloading one or all depots for an app:"); + Console.WriteLine(" depotdownloader -app [-depot [-manifest ]]"); + Console.WriteLine(" [-username [-password ]] [other options]"); Console.WriteLine(); - Console.WriteLine("Usage - downloading a workshop item using pubfile id"); - Console.WriteLine("\tdepotdownloader -app -pubfile [-username [-password ]]"); - Console.WriteLine("Usage - downloading a workshop item using ugc id"); - Console.WriteLine("\tdepotdownloader -app -ugc [-username [-password ]]"); + Console.WriteLine("Usage: downloading a workshop item using pubfile id"); + Console.WriteLine(" depotdownloader -app -pubfile [-username [-password ]]"); + Console.WriteLine("Usage: downloading a workshop item using ugc id"); + Console.WriteLine(" depotdownloader -app -ugc [-username [-password ]]"); Console.WriteLine(); Console.WriteLine("Parameters:"); - Console.WriteLine("\t-app <#>\t\t\t\t- the AppID to download."); - Console.WriteLine("\t-depot <#>\t\t\t\t- the DepotID to download."); - Console.WriteLine("\t-manifest \t\t\t- manifest id of content to download (requires -depot, default: current for branch)."); - Console.WriteLine($"\t-beta \t\t\t- download from specified branch if available (default: {ContentDownloader.DEFAULT_BRANCH})."); - Console.WriteLine("\t-betapassword \t\t- branch password if applicable."); - Console.WriteLine("\t-all-archs\t\t\t- download all architecture-specific depots when -app is used."); - Console.WriteLine("\t-all-platforms\t\t\t- downloads all platform-specific depots when -app is used."); - Console.WriteLine("\t-os \t\t\t\t- the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on)"); - Console.WriteLine("\t-osarch \t\t\t\t- the architecture for which to download the game (32 or 64, default: the host's architecture)"); - Console.WriteLine("\t-all-languages\t\t\t\t- download all language-specific depots when -app is used."); - Console.WriteLine("\t-language \t\t\t\t- the language for which to download the game (default: english)"); - Console.WriteLine("\t-lowviolence\t\t\t\t- download low violence depots when -app is used."); + Console.WriteLine(" -app <#> - the AppID to download."); + Console.WriteLine(" -depot <#> - the DepotID to download."); + Console.WriteLine(" -manifest - manifest id of content to download (requires -depot, default: current for branch)."); + Console.WriteLine($" -beta - download from specified branch if available (default: {ContentDownloader.DEFAULT_BRANCH})."); + Console.WriteLine(" -betapassword - branch password if applicable."); + Console.WriteLine(" -all-platforms - downloads all platform-specific depots when -app is used."); + Console.WriteLine(" -all-archs - download all architecture-specific depots when -app is used."); + Console.WriteLine(" -os - the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on)"); + Console.WriteLine(" -osarch - the architecture for which to download the game (32 or 64, default: the host's architecture)"); + Console.WriteLine(" -all-languages - download all language-specific depots when -app is used."); + Console.WriteLine(" -language - the language for which to download the game (default: english)"); + Console.WriteLine(" -lowviolence - download low violence depots when -app is used."); Console.WriteLine(); - Console.WriteLine("\t-ugc <#>\t\t\t\t- the UGC ID to download."); - Console.WriteLine("\t-pubfile <#>\t\t\t- the PublishedFileId to download. (Will automatically resolve to UGC id)"); + Console.WriteLine(" -ugc <#> - the UGC ID to download."); + Console.WriteLine(" -pubfile <#> - the PublishedFileId to download. (Will automatically resolve to UGC id)"); Console.WriteLine(); - Console.WriteLine("\t-username \t\t- the username of the account to login to for restricted content."); - Console.WriteLine("\t-password \t\t- the password of the account to login to for restricted content."); - Console.WriteLine("\t-remember-password\t\t- if set, remember the password for subsequent logins of this user. (Use -username -remember-password as login credentials)"); + Console.WriteLine(" -username - the username of the account to login to for restricted content."); + Console.WriteLine(" -password - the password of the account to login to for restricted content."); + Console.WriteLine(" -remember-password - if set, remember the password for subsequent logins of this user. (Use -username -remember-password as login credentials)"); Console.WriteLine(); - Console.WriteLine("\t-dir \t\t- the directory in which to place downloaded files."); - Console.WriteLine("\t-filelist \t- a list of files to download (from the manifest). Prefix file path with 'regex:' if you want to match with regex."); - Console.WriteLine("\t-validate\t\t\t\t- Include checksum verification of files already downloaded"); + Console.WriteLine(" -dir - the directory in which to place downloaded files."); + Console.WriteLine(" -filelist - a list of files to download (from the manifest). Prefix file path with 'regex:' if you want to match with regex."); + Console.WriteLine(" -validate - Include checksum verification of files already downloaded"); Console.WriteLine(); - Console.WriteLine("\t-manifest-only\t\t\t- downloads a human readable manifest for any depots that would be downloaded."); - Console.WriteLine("\t-cellid <#>\t\t\t\t- the overridden CellID of the content server to download from."); - Console.WriteLine("\t-max-servers <#>\t\t- maximum number of content servers to use. (default: 20)."); - Console.WriteLine("\t-max-downloads <#>\t\t- maximum number of chunks to download concurrently. (default: 8)."); - Console.WriteLine("\t-loginid <#>\t\t- a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently."); + Console.WriteLine(" -manifest-only - downloads a human readable manifest for any depots that would be downloaded."); + Console.WriteLine(" -cellid <#> - the overridden CellID of the content server to download from."); + Console.WriteLine(" -max-servers <#> - maximum number of content servers to use. (default: 20)."); + Console.WriteLine(" -max-downloads <#> - maximum number of chunks to download concurrently. (default: 8)."); + Console.WriteLine(" -loginid <#> - a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently."); + } + + static void PrintVersion(bool printExtra = false) + { + var version = typeof(Program).Assembly.GetCustomAttribute().InformationalVersion; + Console.WriteLine($"DepotDownloader v{version}"); + + if (!printExtra) + { + return; + } + + Console.WriteLine($"Runtime: {RuntimeInformation.FrameworkDescription} on {RuntimeInformation.OSDescription}"); } } } diff --git a/DepotDownloader/ProtoManifest.cs b/DepotDownloader/ProtoManifest.cs index e4a9c854..b6eb0100 100644 --- a/DepotDownloader/ProtoManifest.cs +++ b/DepotDownloader/ProtoManifest.cs @@ -1,3 +1,6 @@ +// This file is subject to the terms and conditions defined +// in file 'LICENSE', which is part of this source code package. + using System; using System.Collections.Generic; using System.IO; @@ -76,7 +79,7 @@ public class ChunkData public ChunkData(DepotManifest.ChunkData sourceChunk) { ChunkID = sourceChunk.ChunkID; - Checksum = sourceChunk.Checksum; + Checksum = BitConverter.GetBytes(sourceChunk.Checksum); Offset = sourceChunk.Offset; CompressedLength = sourceChunk.CompressedLength; UncompressedLength = sourceChunk.UncompressedLength; diff --git a/DepotDownloader/Steam3Session.cs b/DepotDownloader/Steam3Session.cs index d2ee88ab..906b309d 100644 --- a/DepotDownloader/Steam3Session.cs +++ b/DepotDownloader/Steam3Session.cs @@ -1,4 +1,8 @@ +// This file is subject to the terms and conditions defined +// in file 'LICENSE', which is part of this source code package. + using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Linq; @@ -7,6 +11,7 @@ using QRCoder; using SteamKit2; using SteamKit2.Authentication; +using SteamKit2.CDN; using SteamKit2.Internal; namespace DepotDownloader @@ -24,6 +29,7 @@ public ReadOnlyCollection Licenses public Dictionary AppTokens { get; } = []; public Dictionary PackageTokens { get; } = []; public Dictionary DepotKeys { get; } = []; + public ConcurrentDictionary<(uint, string), TaskCompletionSource> CDNAuthTokens { get; } = []; public Dictionary AppInfo { get; } = []; public Dictionary PackageInfo { get; } = []; public Dictionary AppBetaPasswords { get; } = []; @@ -282,6 +288,30 @@ public async Task GetDepotManifestRequestCodeAsync(uint depotId, uint app return requestCode; } + public async Task RequestCDNAuthToken(uint appid, uint depotid, Server server) + { + var cdnKey = (depotid, server.Host); + var completion = new TaskCompletionSource(); + + if (bAborted || !CDNAuthTokens.TryAdd(cdnKey, completion)) + { + return; + } + + DebugLog.WriteLine(nameof(Steam3Session), $"Requesting CDN auth token for {server.Host}"); + + var cdnAuth = await steamApps.GetCDNAuthToken(appid, depotid, server.Host); + + Console.WriteLine($"Got CDN auth token for {server.Host} result: {cdnAuth.Result} (expires {cdnAuth.Expiration})"); + + if (cdnAuth.Result != EResult.OK) + { + return; + } + + completion.TrySetResult(cdnAuth); + } + public void CheckAppBetaPassword(uint appid, string password) { var completed = false; @@ -403,6 +433,8 @@ public void Disconnect(bool sendLogOff = true) bIsConnectionRecovery = false; steamClient.Disconnect(); + Ansi.Progress(Ansi.ProgressState.Hidden); + // flush callbacks until our disconnected event while (!bDidDisconnect) { diff --git a/DepotDownloader/Util.cs b/DepotDownloader/Util.cs index d249c1fc..46045ae4 100644 --- a/DepotDownloader/Util.cs +++ b/DepotDownloader/Util.cs @@ -1,8 +1,12 @@ +// This file is subject to the terms and conditions defined +// in file 'LICENSE', which is part of this source code package. + using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Runtime.InteropServices; +using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; @@ -27,6 +31,12 @@ public static string GetSteamOS() return "linux"; } + if (RuntimeInformation.IsOSPlatform(OSPlatform.FreeBSD)) + { + // Return linux as freebsd steam client doesn't exist yet + return "linux"; + } + return "unknown"; } @@ -71,26 +81,12 @@ public static string ReadPassword() public static List ValidateSteam3FileChecksums(FileStream fs, ProtoManifest.ChunkData[] chunkdata) { var neededChunks = new List(); - int read; foreach (var data in chunkdata) { - var chunk = new byte[data.UncompressedLength]; fs.Seek((long)data.Offset, SeekOrigin.Begin); - read = fs.Read(chunk, 0, (int)data.UncompressedLength); - byte[] tempchunk; - if (read < data.UncompressedLength) - { - tempchunk = new byte[read]; - Array.Copy(chunk, 0, tempchunk, 0, read); - } - else - { - tempchunk = chunk; - } - - var adler = AdlerHash(tempchunk); + var adler = AdlerHash(fs, (int)data.UncompressedLength); if (!adler.SequenceEqual(data.Checksum)) { neededChunks.Add(data); @@ -100,12 +96,14 @@ public static string ReadPassword() return neededChunks; } - public static byte[] AdlerHash(byte[] input) + public static byte[] AdlerHash(Stream stream, int length) { uint a = 0, b = 0; - for (var i = 0; i < input.Length; i++) + for (var i = 0; i < length; i++) { - a = (a + input[i]) % 65521; + var c = (uint)stream.ReadByte(); + + a = (a + c) % 65521; b = (b + a) % 65521; } @@ -126,11 +124,21 @@ public static byte[] DecodeHexString(string hex) return bytes; } - public static string EncodeHexString(byte[] input) + /// + /// Decrypts using AES/ECB/PKCS7 + /// + public static byte[] SymmetricDecryptECB(byte[] input, byte[] key) { - return input.Aggregate(new StringBuilder(), - (sb, v) => sb.Append(v.ToString("x2")) - ).ToString(); + using var aes = Aes.Create(); + aes.BlockSize = 128; + aes.KeySize = 256; + aes.Mode = CipherMode.ECB; + aes.Padding = PaddingMode.PKCS7; + + using var aesTransform = aes.CreateDecryptor(key, null); + var output = aesTransform.TransformFinalBlock(input, 0, input.Length); + + return output; } public static async Task InvokeAsync(IEnumerable> taskFactories, int maxDegreeOfParallelism) diff --git a/README.md b/README.md index 02d75ed3..4a0740d0 100644 --- a/README.md +++ b/README.md @@ -3,58 +3,92 @@ DepotDownloader Steam depot downloader utilizing the SteamKit2 library. Supports .NET 8.0 -### Downloading one or all depots for an app +This program must be run from a console, it has no GUI. + +## Installation + +### Directly from GitHub + +Download a binary from [the releases page](https://github.com/SteamRE/DepotDownloader/releases/latest). + +### via Windows Package Manager CLI (aka winget) + +On Windows, [winget](https://github.com/microsoft/winget-cli) users can download and install +the latest Terminal release by installing the `SteamRE.DepotDownloader` +package: + +```powershell +winget install --exact --id SteamRE.DepotDownloader ``` -dotnet DepotDownloader.dll -app [-depot [-manifest ]] - [-username [-password ]] [other options] + +### via Homebrew + +On macOS, [Homebrew](https://brew.sh) users can download and install that latest release by running the following commands: + +```shell +brew tap steamre/tools +brew install depotdownloader ``` -For example: `dotnet DepotDownloader.dll -app 730 -depot 731 -manifest 7617088375292372759` +## Usage -### Downloading a workshop item using pubfile id +### Downloading one or all depots for an app +```powershell +./DepotDownloader -app [-depot [-manifest ]] + [-username [-password ]] [other options] ``` -dotnet DepotDownloader.dll -app -pubfile [-username [-password ]] + +For example: `./DepotDownloader -app 730 -depot 731 -manifest 7617088375292372759` + +By default it will use anonymous account ([view which apps are available on it here](https://steamdb.info/sub/17906/)). + +To use your account, specify the `-username ` parameter. Password will be asked interactively if you do +not use specify the `-password` parameter. + +### Downloading a workshop item using pubfile id +```powershell +./DepotDownloader -app -pubfile [-username [-password ]] ``` -For example: `dotnet DepotDownloader.dll -app 730 -pubfile 1885082371` +For example: `./DepotDownloader -app 730 -pubfile 1885082371` ### Downloading a workshop item using ugc id -``` -dotnet DepotDownloader.dll -app -ugc [-username [-password ]] +```powershell +./DepotDownloader -app -ugc [-username [-password ]] ``` -For example: `dotnet DepotDownloader.dll -app 730 -ugc 770604181014286929` +For example: `./DepotDownloader -app 730 -ugc 770604181014286929` ## Parameters -Parameter | Description ---------- | ----------- --app \<#> | the AppID to download. --depot \<#> | the DepotID to download. --manifest \ | manifest id of content to download (requires -depot, default: current for branch). --ugc \<#> | the UGC ID to download. --beta \ | download from specified branch if available (default: Public). --betapassword \ | branch password if applicable. --all-platforms | downloads all platform-specific depots when -app is used. --os \ | the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on) --osarch \ | the architecture for which to download the game (32 or 64, default: the host's architecture) --all-archs | download all architecture-specific depots when -app is used. --all-languages | download all language-specific depots when -app is used. --language \ | the language for which to download the game (default: english) --lowviolence | download low violence depots when -app is used. --pubfile \<#> | the PublishedFileId to download. (Will automatically resolve to UGC id) --username \ | the username of the account to login to for restricted content. --password \ | the password of the account to login to for restricted content. --remember-password | if set, remember the password for subsequent logins of this user. (Use -username -remember-password as login credentials) --dir \ | the directory in which to place downloaded files. --filelist \ | a list of files to download (from the manifest). Prefix file path with `regex:` if you want to match with regex. --validate | Include checksum verification of files already downloaded --manifest-only | downloads a human readable manifest for any depots that would be downloaded. --cellid \<#> | the overridden CellID of the content server to download from. --max-servers \<#> | maximum number of content servers to use. (default: 20). --max-downloads \<#> | maximum number of chunks to download concurrently. (default: 8). --loginid \<#> | a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently. - +Parameter | Description +----------------------- | ----------- +`-app <#>` | the AppID to download. +`-depot <#>` | the DepotID to download. +`-manifest ` | manifest id of content to download (requires `-depot`, default: current for branch). +`-ugc <#>` | the UGC ID to download. +`-beta ` | download from specified branch if available (default: Public). +`-betapassword ` | branch password if applicable. +`-all-platforms` | downloads all platform-specific depots when `-app` is used. +`-os ` | the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on) +`-osarch ` | the architecture for which to download the game (32 or 64, default: the host's architecture) +`-all-archs` | download all architecture-specific depots when `-app` is used. +`-all-languages` | download all language-specific depots when `-app` is used. +`-language ` | the language for which to download the game (default: english) +`-lowviolence` | download low violence depots when `-app` is used. +`-pubfile <#>` | the PublishedFileId to download. (Will automatically resolve to UGC id) +`-username ` | the username of the account to login to for restricted content. +`-password ` | the password of the account to login to for restricted content. +`-remember-password` | if set, remember the password for subsequent logins of this user. (Use `-username -remember-password` as login credentials) +`-dir ` | the directory in which to place downloaded files. +`-filelist ` | a list of files to download (from the manifest). Prefix file path with `regex:` if you want to match with regex. +`-validate` | Include checksum verification of files already downloaded +`-manifest-only` | downloads a human readable manifest for any depots that would be downloaded. +`-cellid <#>` | the overridden CellID of the content server to download from. +`-max-servers <#>` | maximum number of content servers to use. (default: 20). +`-max-downloads <#>` | maximum number of chunks to download concurrently. (default: 8). +`-loginid <#>` | a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently. +`-V` or `--version` | print version and runtime ## Frequently Asked Questions