Create skip-releases option for NetKAN

With the --skip-releases <int> option set, NetKAN skips the given number of releases for inflation. This happens in the transformers for all the kref sources, just before the --releases option is applied to take multiple releases. This feature is useful for backfilling old releases of a mod, where you don't want to re-inflate the newer ones just to get to the older ones.
KSP-CKAN · Feb 10, 2020 · 58c84a1 · 58c84a1
1 parent 33a0fbc
commit 58c84a1
Show file tree

Hide file tree

Showing 23 changed files with 47 additions and 19 deletions.
diff --git a/Netkan/CmdLineOptions.cs b/Netkan/CmdLineOptions.cs
@@ -31,6 +31,9 @@ internal class CmdLineOptions
         [Option("releases", DefaultValue = "1", HelpText = "Number of releases to inflate, or 'all'")]
         public string Releases { get; set; }
 
+        [Option("skip-releases", DefaultValue = "0", HelpText = "Number of releases to skip / index of release to inflate.")]
+        public string SkipReleases { get; set; }
+
         [Option("prerelease", HelpText = "Index GitHub prereleases")]
         public bool PreRelease { get; set; }
 

diff --git a/Netkan/Processors/QueueHandler.cs b/Netkan/Processors/QueueHandler.cs
@@ -116,7 +116,7 @@ private IEnumerable<SendMessageBatchRequestEntry> Inflate(Message msg)
             IEnumerable<Metadata> ckans = null;
             bool   caught        = false;
             string caughtMessage = null;
-            var    opts          = new TransformOptions(releases, highVer);
+            var    opts          = new TransformOptions(releases, null, highVer);
             try
             {
                 ckans = inflator.Inflate($"{netkan.Identifier}.netkan", netkan, opts);
@@ -147,7 +147,7 @@ private IEnumerable<SendMessageBatchRequestEntry> Inflate(Message msg)
         private SendMessageBatchRequestEntry inflationMessage(Metadata ckan, Metadata netkan, TransformOptions opts, bool success, string err = null)
         {
             bool staged = netkan.Staged || opts.Staged;
-            string stagingReason = 
+            string stagingReason =
                   !string.IsNullOrEmpty(netkan.StagingReason) ? netkan.StagingReason
                 : !string.IsNullOrEmpty(opts.StagingReason)   ? opts.StagingReason
                 : null;

diff --git a/Netkan/Program.cs b/Netkan/Program.cs
@@ -93,6 +93,7 @@ public static int Main(string[] args)
                         netkan,
                         new TransformOptions(
                             ParseReleases(Options.Releases),
+                            ParseSkipReleases(Options.SkipReleases),
                             ParseHighestVersion(Options.HighestVersion)
                         )
                     );
@@ -131,6 +132,11 @@ public static int Main(string[] args)
             return val == "all" ? (int?)null : int.Parse(val);
         }
 
+        private static int? ParseSkipReleases(string val)
+        {
+            return string.IsNullOrWhiteSpace(val) ? (int?)null : int.Parse(val);
+        }
+
         private static ModuleVersion ParseHighestVersion(string val)
         {
             return val == null ? null : new ModuleVersion(val);

diff --git a/Netkan/Transformers/CurseTransformer.cs b/Netkan/Transformers/CurseTransformer.cs
@@ -39,6 +39,10 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
                 // Look up our mod on Curse by its Id.
                 var curseMod = _api.GetMod(metadata.Kref.Id);
                 var versions = curseMod.All();
+                if (opts.SkipReleases.HasValue)
+                {
+                    versions = versions.Skip(opts.SkipReleases.Value);
+                }
                 if (opts.Releases.HasValue)
                 {
                     versions = versions.Take(opts.Releases.Value);

diff --git a/Netkan/Transformers/GithubTransformer.cs b/Netkan/Transformers/GithubTransformer.cs
@@ -57,6 +57,10 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
                 // Get the GitHub repository
                 var ghRepo = _api.GetRepo(ghRef);
                 var versions = _api.GetAllReleases(ghRef);
+                if (opts.SkipReleases.HasValue)
+                {
+                    versions = versions.Skip(opts.SkipReleases.Value);
+                }
                 if (opts.Releases.HasValue)
                 {
                     versions = versions.Take(opts.Releases.Value);
@@ -168,7 +172,7 @@ private JToken getAuthors(GithubRepo repo, GithubRelease release)
                 }
                 // Check parent repos
                 r = r.ParentRepo == null
-                    ? null 
+                    ? null
                     : _api.GetRepo(new GithubRef($"#/ckan/github/{r.ParentRepo.FullName}", false, _matchPreleases));
             }
             // Return a string if just one author, else an array

diff --git a/Netkan/Transformers/ITransformer.cs b/Netkan/Transformers/ITransformer.cs
@@ -6,13 +6,15 @@ namespace CKAN.NetKAN.Transformers
 {
     internal class TransformOptions
     {
-        public TransformOptions(int? releases, ModuleVersion highVer)
+        public TransformOptions(int? releases, int? skipReleases, ModuleVersion highVer)
         {
             Releases       = releases;
+            SkipReleases   = skipReleases;
             HighestVersion = highVer;
         }
 
         public readonly int?          Releases;
+        public readonly int?          SkipReleases;
         public readonly ModuleVersion HighestVersion;
         public          bool          Staged;
         public          string        StagingReason;

diff --git a/Netkan/Transformers/JenkinsTransformer.cs b/Netkan/Transformers/JenkinsTransformer.cs
@@ -35,6 +35,10 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
                 JenkinsRef jRef = new JenkinsRef(metadata.Kref);
 
                 var versions = _api.GetAllBuilds(jRef, options);
+                if (opts.SkipReleases.HasValue)
+                {
+                    versions = versions.Skip(opts.SkipReleases.Value);
+                }
                 if (opts.Releases.HasValue)
                 {
                     versions = versions.Take(opts.Releases.Value);

diff --git a/Netkan/Transformers/SpacedockTransformer.cs b/Netkan/Transformers/SpacedockTransformer.cs
@@ -38,6 +38,10 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
                 // Look up our mod on SD by its Id.
                 var sdMod = _api.GetMod(Convert.ToInt32(metadata.Kref.Id));
                 var versions = sdMod.All();
+                if (opts.SkipReleases.HasValue)
+                {
+                    versions = versions.Skip(opts.SkipReleases.Value);
+                }
                 if (opts.Releases.HasValue)
                 {
                     versions = versions.Take(opts.Releases.Value);

diff --git a/Tests/NetKAN/MainClass.cs b/Tests/NetKAN/MainClass.cs
@@ -12,7 +12,7 @@ namespace Tests.NetKAN
     [TestFixture]
     public class MainClassTests
     {
-        private TransformOptions opts = new TransformOptions(1, null);
+        private TransformOptions opts = new TransformOptions(1, null, null);
 
         [Test]
         public void FixVersionStringsUnharmed()

diff --git a/Tests/NetKAN/NetkanOverride.cs b/Tests/NetKAN/NetkanOverride.cs
@@ -11,7 +11,7 @@ namespace Tests.NetKAN
     public class NetkanOverride
     {
         JObject such_metadata;
-        private TransformOptions opts = new TransformOptions(1, null);
+        private TransformOptions opts = new TransformOptions(1, null, null);
 
         [SetUp]
         public void Setup()

diff --git a/Tests/NetKAN/Transformers/AvcKrefTransformerTests.cs b/Tests/NetKAN/Transformers/AvcKrefTransformerTests.cs
@@ -12,7 +12,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class AvcKrefTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1, null);
+        private TransformOptions opts = new TransformOptions(1, null, null);
 
         [Test,
             TestCase(

diff --git a/Tests/NetKAN/Transformers/AvcTransformerTests.cs b/Tests/NetKAN/Transformers/AvcTransformerTests.cs
@@ -14,7 +14,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class AvcTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1, null);
+        private TransformOptions opts = new TransformOptions(1, null, null);
 
         [Test]
         public void AddsMissingVersionInfo()

diff --git a/Tests/NetKAN/Transformers/CurseTransformerTests.cs b/Tests/NetKAN/Transformers/CurseTransformerTests.cs
@@ -12,7 +12,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class CurseTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1, null);
+        private TransformOptions opts = new TransformOptions(1, null, null);
 
         // GH #199: Don't pre-fill KSP version fields if we see a ksp_min/max
         [Test]

diff --git a/Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs b/Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs
@@ -13,7 +13,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class DownloadAttributeTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1, null);
+        private TransformOptions opts = new TransformOptions(1, null, null);
 
         [Test]
         public void AddsDownloadAttributes()

diff --git a/Tests/NetKAN/Transformers/EpochTransformerTests.cs b/Tests/NetKAN/Transformers/EpochTransformerTests.cs
@@ -33,6 +33,7 @@ public void Transform_WithHighVersionParam_MatchesExpected(string version, strin
             ITransformer     sut  = new EpochTransformer();
             TransformOptions opts = new TransformOptions(
                 1,
+                null,
                 string.IsNullOrEmpty(highVer)
                     ? null
                     : new ModuleVersion(highVer)

diff --git a/Tests/NetKAN/Transformers/GeneratedByTransformerTests.cs b/Tests/NetKAN/Transformers/GeneratedByTransformerTests.cs
@@ -9,8 +9,8 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class GeneratedByTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1, null);
-        
+        private TransformOptions opts = new TransformOptions(1, null, null);
+
         [Test]
         public void AddsGeneratedByProperty()
         {

diff --git a/Tests/NetKAN/Transformers/GithubTransformerTests.cs b/Tests/NetKAN/Transformers/GithubTransformerTests.cs
@@ -14,7 +14,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class GithubTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1, null);
+        private TransformOptions opts = new TransformOptions(1, null, null);
 
         [Test]
         public void CalculatesRepositoryUrlCorrectly()

diff --git a/Tests/NetKAN/Transformers/HttpTransformerTests.cs b/Tests/NetKAN/Transformers/HttpTransformerTests.cs
@@ -9,7 +9,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class HttpTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1, null);
+        private TransformOptions opts = new TransformOptions(1, null, null);
 
         [TestCase("#/ckan/github/foo/bar")]
         [TestCase("#/ckan/netkan/http://awesomemod.example/awesomemod.netkan")]

diff --git a/Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs b/Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs
@@ -12,7 +12,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class InternalCkanTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1, null);
+        private TransformOptions opts = new TransformOptions(1, null, null);
 
         [Test]
         public void AddsMiddingProperties()

diff --git a/Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs b/Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs
@@ -13,7 +13,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class MetaNetkanTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1, null);
+        private TransformOptions opts = new TransformOptions(1, null, null);
 
         [Test]
         public void DoesNothingWhenNoMatch()

diff --git a/Tests/NetKAN/Transformers/SpacedockTransformerTests.cs b/Tests/NetKAN/Transformers/SpacedockTransformerTests.cs
@@ -14,7 +14,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class SpacedockTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1, null);
+        private TransformOptions opts = new TransformOptions(1, null, null);
 
         // GH #199: Don't pre-fill KSP version fields if we see a ksp_min/max
         [Test]

diff --git a/Tests/NetKAN/Transformers/StripNetkanMetadataTransformerTests.cs b/Tests/NetKAN/Transformers/StripNetkanMetadataTransformerTests.cs
@@ -10,7 +10,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class StripNetkanMetadataTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1, null);
+        private TransformOptions opts = new TransformOptions(1, null, null);
 
         [TestCaseSource("StripNetkanMetadataTestCaseSource")]
         public void StripNetkanMetadata(string json, string expected)

diff --git a/Tests/NetKAN/Transformers/VersionEditTransformerTests.cs b/Tests/NetKAN/Transformers/VersionEditTransformerTests.cs
@@ -10,7 +10,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class VersionEditTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1, null);
+        private TransformOptions opts = new TransformOptions(1, null, null);
 
         [Test]
         public void DoesNothingWhenNoMatch()