From 58c84a12917e5d085a7ac962d06bb50a29a458fb Mon Sep 17 00:00:00 2001 From: DasSkelett Date: Mon, 10 Feb 2020 18:37:17 +0100 Subject: [PATCH] Create skip-releases option for NetKAN With the --skip-releases option set, NetKAN skips the given number of releases for inflation. This happens in the transformers for all the kref sources, just before the --releases option is applied to take multiple releases. This feature is useful for backfilling old releases of a mod, where you don't want to re-inflate the newer ones just to get to the older ones. --- Netkan/CmdLineOptions.cs | 3 +++ Netkan/Processors/QueueHandler.cs | 4 ++-- Netkan/Program.cs | 6 ++++++ Netkan/Transformers/CurseTransformer.cs | 4 ++++ Netkan/Transformers/GithubTransformer.cs | 6 +++++- Netkan/Transformers/ITransformer.cs | 4 +++- Netkan/Transformers/JenkinsTransformer.cs | 4 ++++ Netkan/Transformers/SpacedockTransformer.cs | 4 ++++ Tests/NetKAN/MainClass.cs | 2 +- Tests/NetKAN/NetkanOverride.cs | 2 +- Tests/NetKAN/Transformers/AvcKrefTransformerTests.cs | 2 +- Tests/NetKAN/Transformers/AvcTransformerTests.cs | 2 +- Tests/NetKAN/Transformers/CurseTransformerTests.cs | 2 +- .../Transformers/DownloadAttributeTransformerTests.cs | 2 +- Tests/NetKAN/Transformers/EpochTransformerTests.cs | 1 + Tests/NetKAN/Transformers/GeneratedByTransformerTests.cs | 4 ++-- Tests/NetKAN/Transformers/GithubTransformerTests.cs | 2 +- Tests/NetKAN/Transformers/HttpTransformerTests.cs | 2 +- Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs | 2 +- Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs | 2 +- Tests/NetKAN/Transformers/SpacedockTransformerTests.cs | 2 +- .../Transformers/StripNetkanMetadataTransformerTests.cs | 2 +- Tests/NetKAN/Transformers/VersionEditTransformerTests.cs | 2 +- 23 files changed, 47 insertions(+), 19 deletions(-) diff --git a/Netkan/CmdLineOptions.cs b/Netkan/CmdLineOptions.cs index 08ca1e5756..01fbca5af4 100644 --- a/Netkan/CmdLineOptions.cs +++ b/Netkan/CmdLineOptions.cs @@ -31,6 +31,9 @@ internal class CmdLineOptions [Option("releases", DefaultValue = "1", HelpText = "Number of releases to inflate, or 'all'")] public string Releases { get; set; } + [Option("skip-releases", DefaultValue = "0", HelpText = "Number of releases to skip / index of release to inflate.")] + public string SkipReleases { get; set; } + [Option("prerelease", HelpText = "Index GitHub prereleases")] public bool PreRelease { get; set; } diff --git a/Netkan/Processors/QueueHandler.cs b/Netkan/Processors/QueueHandler.cs index ba3638e10a..66e05386ec 100644 --- a/Netkan/Processors/QueueHandler.cs +++ b/Netkan/Processors/QueueHandler.cs @@ -116,7 +116,7 @@ private IEnumerable Inflate(Message msg) IEnumerable ckans = null; bool caught = false; string caughtMessage = null; - var opts = new TransformOptions(releases, highVer); + var opts = new TransformOptions(releases, null, highVer); try { ckans = inflator.Inflate($"{netkan.Identifier}.netkan", netkan, opts); @@ -147,7 +147,7 @@ private IEnumerable Inflate(Message msg) private SendMessageBatchRequestEntry inflationMessage(Metadata ckan, Metadata netkan, TransformOptions opts, bool success, string err = null) { bool staged = netkan.Staged || opts.Staged; - string stagingReason = + string stagingReason = !string.IsNullOrEmpty(netkan.StagingReason) ? netkan.StagingReason : !string.IsNullOrEmpty(opts.StagingReason) ? opts.StagingReason : null; diff --git a/Netkan/Program.cs b/Netkan/Program.cs index 91075538f6..bf02aed93d 100644 --- a/Netkan/Program.cs +++ b/Netkan/Program.cs @@ -93,6 +93,7 @@ public static int Main(string[] args) netkan, new TransformOptions( ParseReleases(Options.Releases), + ParseSkipReleases(Options.SkipReleases), ParseHighestVersion(Options.HighestVersion) ) ); @@ -131,6 +132,11 @@ public static int Main(string[] args) return val == "all" ? (int?)null : int.Parse(val); } + private static int? ParseSkipReleases(string val) + { + return string.IsNullOrWhiteSpace(val) ? (int?)null : int.Parse(val); + } + private static ModuleVersion ParseHighestVersion(string val) { return val == null ? null : new ModuleVersion(val); diff --git a/Netkan/Transformers/CurseTransformer.cs b/Netkan/Transformers/CurseTransformer.cs index 7a712c755a..00bd4b2e59 100755 --- a/Netkan/Transformers/CurseTransformer.cs +++ b/Netkan/Transformers/CurseTransformer.cs @@ -39,6 +39,10 @@ public IEnumerable Transform(Metadata metadata, TransformOptions opts) // Look up our mod on Curse by its Id. var curseMod = _api.GetMod(metadata.Kref.Id); var versions = curseMod.All(); + if (opts.SkipReleases.HasValue) + { + versions = versions.Skip(opts.SkipReleases.Value); + } if (opts.Releases.HasValue) { versions = versions.Take(opts.Releases.Value); diff --git a/Netkan/Transformers/GithubTransformer.cs b/Netkan/Transformers/GithubTransformer.cs index a56bd93aa0..6541643048 100644 --- a/Netkan/Transformers/GithubTransformer.cs +++ b/Netkan/Transformers/GithubTransformer.cs @@ -57,6 +57,10 @@ public IEnumerable Transform(Metadata metadata, TransformOptions opts) // Get the GitHub repository var ghRepo = _api.GetRepo(ghRef); var versions = _api.GetAllReleases(ghRef); + if (opts.SkipReleases.HasValue) + { + versions = versions.Skip(opts.SkipReleases.Value); + } if (opts.Releases.HasValue) { versions = versions.Take(opts.Releases.Value); @@ -168,7 +172,7 @@ private JToken getAuthors(GithubRepo repo, GithubRelease release) } // Check parent repos r = r.ParentRepo == null - ? null + ? null : _api.GetRepo(new GithubRef($"#/ckan/github/{r.ParentRepo.FullName}", false, _matchPreleases)); } // Return a string if just one author, else an array diff --git a/Netkan/Transformers/ITransformer.cs b/Netkan/Transformers/ITransformer.cs index 06d16ca0ae..e28ce0ceff 100644 --- a/Netkan/Transformers/ITransformer.cs +++ b/Netkan/Transformers/ITransformer.cs @@ -6,13 +6,15 @@ namespace CKAN.NetKAN.Transformers { internal class TransformOptions { - public TransformOptions(int? releases, ModuleVersion highVer) + public TransformOptions(int? releases, int? skipReleases, ModuleVersion highVer) { Releases = releases; + SkipReleases = skipReleases; HighestVersion = highVer; } public readonly int? Releases; + public readonly int? SkipReleases; public readonly ModuleVersion HighestVersion; public bool Staged; public string StagingReason; diff --git a/Netkan/Transformers/JenkinsTransformer.cs b/Netkan/Transformers/JenkinsTransformer.cs index b756654c0c..00c0a42396 100644 --- a/Netkan/Transformers/JenkinsTransformer.cs +++ b/Netkan/Transformers/JenkinsTransformer.cs @@ -35,6 +35,10 @@ public IEnumerable Transform(Metadata metadata, TransformOptions opts) JenkinsRef jRef = new JenkinsRef(metadata.Kref); var versions = _api.GetAllBuilds(jRef, options); + if (opts.SkipReleases.HasValue) + { + versions = versions.Skip(opts.SkipReleases.Value); + } if (opts.Releases.HasValue) { versions = versions.Take(opts.Releases.Value); diff --git a/Netkan/Transformers/SpacedockTransformer.cs b/Netkan/Transformers/SpacedockTransformer.cs index 67a3396b88..8982d76bba 100644 --- a/Netkan/Transformers/SpacedockTransformer.cs +++ b/Netkan/Transformers/SpacedockTransformer.cs @@ -38,6 +38,10 @@ public IEnumerable Transform(Metadata metadata, TransformOptions opts) // Look up our mod on SD by its Id. var sdMod = _api.GetMod(Convert.ToInt32(metadata.Kref.Id)); var versions = sdMod.All(); + if (opts.SkipReleases.HasValue) + { + versions = versions.Skip(opts.SkipReleases.Value); + } if (opts.Releases.HasValue) { versions = versions.Take(opts.Releases.Value); diff --git a/Tests/NetKAN/MainClass.cs b/Tests/NetKAN/MainClass.cs index 23e4d04b56..6d46890c2d 100644 --- a/Tests/NetKAN/MainClass.cs +++ b/Tests/NetKAN/MainClass.cs @@ -12,7 +12,7 @@ namespace Tests.NetKAN [TestFixture] public class MainClassTests { - private TransformOptions opts = new TransformOptions(1, null); + private TransformOptions opts = new TransformOptions(1, null, null); [Test] public void FixVersionStringsUnharmed() diff --git a/Tests/NetKAN/NetkanOverride.cs b/Tests/NetKAN/NetkanOverride.cs index 3c37312427..100ef5f2ed 100644 --- a/Tests/NetKAN/NetkanOverride.cs +++ b/Tests/NetKAN/NetkanOverride.cs @@ -11,7 +11,7 @@ namespace Tests.NetKAN public class NetkanOverride { JObject such_metadata; - private TransformOptions opts = new TransformOptions(1, null); + private TransformOptions opts = new TransformOptions(1, null, null); [SetUp] public void Setup() diff --git a/Tests/NetKAN/Transformers/AvcKrefTransformerTests.cs b/Tests/NetKAN/Transformers/AvcKrefTransformerTests.cs index 519eba5815..46f50a9055 100644 --- a/Tests/NetKAN/Transformers/AvcKrefTransformerTests.cs +++ b/Tests/NetKAN/Transformers/AvcKrefTransformerTests.cs @@ -12,7 +12,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class AvcKrefTransformerTests { - private TransformOptions opts = new TransformOptions(1, null); + private TransformOptions opts = new TransformOptions(1, null, null); [Test, TestCase( diff --git a/Tests/NetKAN/Transformers/AvcTransformerTests.cs b/Tests/NetKAN/Transformers/AvcTransformerTests.cs index f76850b975..fab5a83cda 100644 --- a/Tests/NetKAN/Transformers/AvcTransformerTests.cs +++ b/Tests/NetKAN/Transformers/AvcTransformerTests.cs @@ -14,7 +14,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class AvcTransformerTests { - private TransformOptions opts = new TransformOptions(1, null); + private TransformOptions opts = new TransformOptions(1, null, null); [Test] public void AddsMissingVersionInfo() diff --git a/Tests/NetKAN/Transformers/CurseTransformerTests.cs b/Tests/NetKAN/Transformers/CurseTransformerTests.cs index 0b84d67259..551d620063 100644 --- a/Tests/NetKAN/Transformers/CurseTransformerTests.cs +++ b/Tests/NetKAN/Transformers/CurseTransformerTests.cs @@ -12,7 +12,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class CurseTransformerTests { - private TransformOptions opts = new TransformOptions(1, null); + private TransformOptions opts = new TransformOptions(1, null, null); // GH #199: Don't pre-fill KSP version fields if we see a ksp_min/max [Test] diff --git a/Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs b/Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs index 316d1f8d04..34a3dc0224 100644 --- a/Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs +++ b/Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs @@ -13,7 +13,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class DownloadAttributeTransformerTests { - private TransformOptions opts = new TransformOptions(1, null); + private TransformOptions opts = new TransformOptions(1, null, null); [Test] public void AddsDownloadAttributes() diff --git a/Tests/NetKAN/Transformers/EpochTransformerTests.cs b/Tests/NetKAN/Transformers/EpochTransformerTests.cs index 7d20945ee3..c2b080618e 100644 --- a/Tests/NetKAN/Transformers/EpochTransformerTests.cs +++ b/Tests/NetKAN/Transformers/EpochTransformerTests.cs @@ -33,6 +33,7 @@ public void Transform_WithHighVersionParam_MatchesExpected(string version, strin ITransformer sut = new EpochTransformer(); TransformOptions opts = new TransformOptions( 1, + null, string.IsNullOrEmpty(highVer) ? null : new ModuleVersion(highVer) diff --git a/Tests/NetKAN/Transformers/GeneratedByTransformerTests.cs b/Tests/NetKAN/Transformers/GeneratedByTransformerTests.cs index 761d84096f..4412f04013 100644 --- a/Tests/NetKAN/Transformers/GeneratedByTransformerTests.cs +++ b/Tests/NetKAN/Transformers/GeneratedByTransformerTests.cs @@ -9,8 +9,8 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class GeneratedByTransformerTests { - private TransformOptions opts = new TransformOptions(1, null); - + private TransformOptions opts = new TransformOptions(1, null, null); + [Test] public void AddsGeneratedByProperty() { diff --git a/Tests/NetKAN/Transformers/GithubTransformerTests.cs b/Tests/NetKAN/Transformers/GithubTransformerTests.cs index 31e17e7ae7..f08d0b745e 100644 --- a/Tests/NetKAN/Transformers/GithubTransformerTests.cs +++ b/Tests/NetKAN/Transformers/GithubTransformerTests.cs @@ -14,7 +14,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class GithubTransformerTests { - private TransformOptions opts = new TransformOptions(1, null); + private TransformOptions opts = new TransformOptions(1, null, null); [Test] public void CalculatesRepositoryUrlCorrectly() diff --git a/Tests/NetKAN/Transformers/HttpTransformerTests.cs b/Tests/NetKAN/Transformers/HttpTransformerTests.cs index 9d70efaae1..84e13bfe86 100644 --- a/Tests/NetKAN/Transformers/HttpTransformerTests.cs +++ b/Tests/NetKAN/Transformers/HttpTransformerTests.cs @@ -9,7 +9,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class HttpTransformerTests { - private TransformOptions opts = new TransformOptions(1, null); + private TransformOptions opts = new TransformOptions(1, null, null); [TestCase("#/ckan/github/foo/bar")] [TestCase("#/ckan/netkan/http://awesomemod.example/awesomemod.netkan")] diff --git a/Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs b/Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs index 627133d355..a98c2c542e 100644 --- a/Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs +++ b/Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs @@ -12,7 +12,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class InternalCkanTransformerTests { - private TransformOptions opts = new TransformOptions(1, null); + private TransformOptions opts = new TransformOptions(1, null, null); [Test] public void AddsMiddingProperties() diff --git a/Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs b/Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs index d58e6abb8f..abe2c5c3c2 100644 --- a/Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs +++ b/Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs @@ -13,7 +13,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class MetaNetkanTransformerTests { - private TransformOptions opts = new TransformOptions(1, null); + private TransformOptions opts = new TransformOptions(1, null, null); [Test] public void DoesNothingWhenNoMatch() diff --git a/Tests/NetKAN/Transformers/SpacedockTransformerTests.cs b/Tests/NetKAN/Transformers/SpacedockTransformerTests.cs index f5baf1364b..b7191da8d3 100644 --- a/Tests/NetKAN/Transformers/SpacedockTransformerTests.cs +++ b/Tests/NetKAN/Transformers/SpacedockTransformerTests.cs @@ -14,7 +14,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class SpacedockTransformerTests { - private TransformOptions opts = new TransformOptions(1, null); + private TransformOptions opts = new TransformOptions(1, null, null); // GH #199: Don't pre-fill KSP version fields if we see a ksp_min/max [Test] diff --git a/Tests/NetKAN/Transformers/StripNetkanMetadataTransformerTests.cs b/Tests/NetKAN/Transformers/StripNetkanMetadataTransformerTests.cs index 472fdebdec..9c1bb1b94f 100644 --- a/Tests/NetKAN/Transformers/StripNetkanMetadataTransformerTests.cs +++ b/Tests/NetKAN/Transformers/StripNetkanMetadataTransformerTests.cs @@ -10,7 +10,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class StripNetkanMetadataTransformerTests { - private TransformOptions opts = new TransformOptions(1, null); + private TransformOptions opts = new TransformOptions(1, null, null); [TestCaseSource("StripNetkanMetadataTestCaseSource")] public void StripNetkanMetadata(string json, string expected) diff --git a/Tests/NetKAN/Transformers/VersionEditTransformerTests.cs b/Tests/NetKAN/Transformers/VersionEditTransformerTests.cs index d5ac5c5732..d27164a586 100644 --- a/Tests/NetKAN/Transformers/VersionEditTransformerTests.cs +++ b/Tests/NetKAN/Transformers/VersionEditTransformerTests.cs @@ -10,7 +10,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class VersionEditTransformerTests { - private TransformOptions opts = new TransformOptions(1, null); + private TransformOptions opts = new TransformOptions(1, null, null); [Test] public void DoesNothingWhenNoMatch()