[prism][java] Update Prism locator to match Python SDK semantics. (#3…

…2619)

* Update PrismLocator to match the python SDK semantics.

* Update validate beam release with prism RC validation instructions.

* rename resolveLocation to resolveSource

---------

Co-authored-by: lostluck <[email protected]>

runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismLocator.java

@@ -54,34 +54,61 @@ class PrismLocator {
  private static final String PRISM_BIN_PATH = ".apache_beam/cache/prism/bin";
  private static final Set<PosixFilePermission> PERMS =
  PosixFilePermissions.fromString("rwxr-xr-x");
- private static final String GITHUB_DOWNLOAD_PREFIX =
-  "https://github.com/apache/beam/releases/download";
- private static final String GITHUB_TAG_PREFIX = "https://github.com/apache/beam/releases/tag";
+ private static final String GITHUB_COMMON_PREFIX = "https://github.com/apache/beam/releases/";
+ private static final String GITHUB_DOWNLOAD_PREFIX = GITHUB_COMMON_PREFIX + "download";
+ private static final String GITHUB_TAG_PREFIX = GITHUB_COMMON_PREFIX + "tag";
 
  private final PrismPipelineOptions options;
 
  PrismLocator(PrismPipelineOptions options) {
  this.options = options;
  }
 
+ String resolveSource() {
+ String from =
+ String.format(
+ "%s/v%s/%s.zip", GITHUB_DOWNLOAD_PREFIX, RELEASE_INFO.getSdkVersion(), buildFileName());
+
+ if (Strings.isNullOrEmpty(options.getPrismLocation())) {
+ return from;
+ }
+ from = options.getPrismLocation();
+
+ // Likely a local file, return it directly.
+ if (!from.startsWith("http")) {
+ return from;
+ }
+
+ // Validate that it's from a safe location: A Beam Github Release
+ checkArgument(
+ options.getPrismLocation().startsWith(GITHUB_COMMON_PREFIX),
+ "Provided --prismLocation URL is not an Apache Beam Github "
+ + "Release page URL or download URL: ",
+ options.getPrismLocation());
+
+ from = options.getPrismLocation();
+
+ // If this is the tag prefix, then build the release download with the version
+ // from the given url.
+ if (options.getPrismLocation().startsWith(GITHUB_TAG_PREFIX)) {
+ Path tagPath = Paths.get(options.getPrismLocation());
+ Path locVersion = tagPath.getName(tagPath.getNameCount() - 1);
+ // The "v" prefix is already included in the version name segment.
+ from = String.format("%s/%s/%s.zip", GITHUB_DOWNLOAD_PREFIX, locVersion, buildFileName());
+ }
+ checkArgument(
+ from.startsWith(GITHUB_DOWNLOAD_PREFIX),
+ "Provided --prismLocation URL could not be resolved to a download URL. ",
+ options.getPrismLocation());
+ return from;
+ }
+
  /**
  * Downloads and prepares a Prism executable for use with the {@link PrismRunner}. The returned
  * {@link String} is the absolute path to the Prism executable.
  */
  String resolve() throws IOException {
-
- String from =
- String.format("%s/v%s/%s.zip", GITHUB_DOWNLOAD_PREFIX, getSDKVersion(), buildFileName());
-
- if (!Strings.isNullOrEmpty(options.getPrismLocation())) {
- checkArgument(
- !options.getPrismLocation().startsWith(GITHUB_TAG_PREFIX),
- "Provided --prismLocation URL is not an Apache Beam Github "
- + "Release page URL or download URL: ",
- from);
-
- from = options.getPrismLocation();
- }
+ String from = resolveSource();
 
  String fromFileName = getNameWithoutExtension(from);
  Path to = Paths.get(userHome(), PRISM_BIN_PATH, fromFileName);
@@ -135,11 +162,6 @@ private String resolve(Path from, Path to) throws IOException {
  return to.toString();
  }
 
- String buildFileName() {
- String version = getSDKVersion();
- return String.format("apache_beam-v%s-prism-%s-%s", version, os(), arch());
- }
-
  private static void unzip(URL from, Path to) {
  try {
  unzip(from.openStream(), to);
@@ -181,6 +203,11 @@ private static String getNameWithoutExtension(String path) {
  .getNameWithoutExtension(path);
  }
 
+ private String buildFileName() {
+ String version = getSDKVersion();
+ return String.format("apache_beam-v%s-prism-%s-%s", version, os(), arch());
+ }
+
  private String getSDKVersion() {
  if (Strings.isNullOrEmpty(options.getPrismVersionOverride())) {
  return RELEASE_INFO.getSdkVersion();

runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismPipelineOptions.java

@@ -32,16 +32,17 @@ public interface PrismPipelineOptions extends PortablePipelineOptions {
  @Description(
  "Path or URL to a prism binary, or zipped binary for the current "
  + "platform (Operating System and Architecture). May also be an Apache "
- + "Beam Github Release page URL, with a matching --prismVersionOverride "
- + "set. This option overrides all others for finding a prism binary.")
+ + "Beam Github Release page URL, which be used to construct download URL for "
+ + " the current platform. ")
  String getPrismLocation();
 
  void setPrismLocation(String prismLocation);
 
  @Description(
  "Override the SDK's version for deriving the Github Release URLs for "
  + "downloading a zipped prism binary, for the current platform. If "
- + "set to a Github Release page URL, then it will use that release page as a base when constructing the download URL.")
+ + "the --prismLocation flag is set to a Github Release page URL, "
+ + "then it will use that release page as a base when constructing the download URL.")
  String getPrismVersionOverride();
 
  void setPrismVersionOverride(String prismVersionOverride);

runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismLocatorTest.java

@@ -30,6 +30,7 @@
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.util.ReleaseInfo;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -40,6 +41,7 @@
 @RunWith(JUnit4.class)
 public class PrismLocatorTest {
 
+ private static final ReleaseInfo RELEASE_INFO = ReleaseInfo.getReleaseInfo();
  private static final Path DESTINATION_DIRECTORY = prismBinDirectory();
 
  @Before
@@ -61,61 +63,102 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
  }
 
  @Test
- public void givenVersionOverride_thenResolves() throws IOException {
- assertThat(Files.exists(DESTINATION_DIRECTORY)).isFalse();
+ public void givenVersionOverride_thenResolvesLocation() throws IOException {
+ PrismPipelineOptions options = options();
+ options.setPrismVersionOverride("2.57.0-RC1");
+
+ PrismLocator underTest = new PrismLocator(options);
+ String got = underTest.resolveSource();
+
+ assertThat(got)
+ .contains(
+ "https://github.com/apache/beam/releases/download/v" + RELEASE_INFO.getSdkVersion());
+ assertThat(got).contains("apache_beam-v2.57.0-RC1-prism");
+ assertThat(got).contains(".zip");
+ }
+
+ // This testcase validates a user override to download a different release version specifically.
+ @Test
+ public void givenHttpPrismLocationOption_thenResolvesLocation() throws IOException {
+ PrismPipelineOptions options = options();
+ String want =
+ "https://github.com/apache/beam/releases/download/v2.57.0/apache_beam-v2.57.0-prism-darwin-arm64.zip";
+ options.setPrismLocation(want);
+
+ PrismLocator underTest = new PrismLocator(options);
+ String got = underTest.resolveSource();
+
+ assertThat(got).isEqualTo(want);
+ }
+
+ // This testcase is the Release Validation behavior, where we provide an RC option, but
+ // need to resolve the download for the non-RC version.
+ // Copy the URL directly, and set the location, override the file's RC version with the final
+ // version.
+ @Test
+ public void givenRCGithubTagPrismLocationOption_thenResolvesLocation() {
  PrismPipelineOptions options = options();
+ options.setPrismLocation("https://github.com/apache/beam/releases/tag/v2.57.0-RC1/");
  options.setPrismVersionOverride("2.57.0");
+
  PrismLocator underTest = new PrismLocator(options);
- String got = underTest.resolve();
- assertThat(got).contains(DESTINATION_DIRECTORY.toString());
- assertThat(got).contains("2.57.0");
- Path gotPath = Paths.get(got);
- assertThat(Files.exists(gotPath)).isTrue();
+ String got = underTest.resolveSource();
+
+ assertThat(got)
+ .contains(
+ "https://github.com/apache/beam/releases/download/v2.57.0-RC1/apache_beam-v2.57.0-prism");
+ assertThat(got).contains(".zip");
  }
 
  @Test
- public void givenHttpPrismLocationOption_thenResolves() throws IOException {
- assertThat(Files.exists(DESTINATION_DIRECTORY)).isFalse();
+ public void givenRCGithubTagPrismLocationOptionNoTrailingSlash_thenResolvesLocation() {
  PrismPipelineOptions options = options();
- options.setPrismLocation(
- "https://github.com/apache/beam/releases/download/v2.57.0/apache_beam-v2.57.0-prism-darwin-arm64.zip");
+ options.setPrismLocation("https://github.com/apache/beam/releases/tag/v2.57.0-RC2");
+ options.setPrismVersionOverride("2.57.0");
+
  PrismLocator underTest = new PrismLocator(options);
- String got = underTest.resolve();
- assertThat(got).contains(DESTINATION_DIRECTORY.toString());
- Path gotPath = Paths.get(got);
- assertThat(Files.exists(gotPath)).isTrue();
+ String got = underTest.resolveSource();
+
+ assertThat(got)
+ .contains(
+ "https://github.com/apache/beam/releases/download/v2.57.0-RC2/apache_beam-v2.57.0-prism");
+ assertThat(got).contains(".zip");
  }
 
  @Test
  public void givenFilePrismLocationOption_thenResolves() throws IOException {
  assertThat(Files.exists(DESTINATION_DIRECTORY)).isFalse();
  PrismPipelineOptions options = options();
  options.setPrismLocation(getLocalPrismBuildOrIgnoreTest());
+
  PrismLocator underTest = new PrismLocator(options);
  String got = underTest.resolve();
+
  assertThat(got).contains(DESTINATION_DIRECTORY.toString());
  Path gotPath = Paths.get(got);
  assertThat(Files.exists(gotPath)).isTrue();
  }
 
  @Test
- public void givenGithubTagPrismLocationOption_thenThrows() {
+ public void givenIncorrectGithubPrismLocationOption_thenThrows() {
  PrismPipelineOptions options = options();
+ // This is an incorrect github download path. Downloads are under /download/ not tag.
  options.setPrismLocation(
  "https://github.com/apache/beam/releases/tag/v2.57.0/apache_beam-v2.57.0-prism-darwin-amd64.zip");
+
  PrismLocator underTest = new PrismLocator(options);
- IllegalArgumentException error =
- assertThrows(IllegalArgumentException.class, underTest::resolve);
- assertThat(error.getMessage())
- .contains(
- "Provided --prismLocation URL is not an Apache Beam Github Release page URL or download URL");
+
+ RuntimeException error = assertThrows(RuntimeException.class, underTest::resolve);
+ // Message should contain the incorrectly constructed download link.
+ assertThat(error.getMessage()).contains(".zip/apache_beam");
  }
 
  @Test
  @Ignore // TODO: use mock site. Currently failing with response code 500 instead of 404
  public void givenPrismLocation404_thenThrows() {
  PrismPipelineOptions options = options();
  options.setPrismLocation("https://example.com/i/dont/exist.zip");
+
  PrismLocator underTest = new PrismLocator(options);
  RuntimeException error = assertThrows(RuntimeException.class, underTest::resolve);
  assertThat(error.getMessage()).contains("NotFoundException");

website/www/site/content/en/blog/validate-beam-release.md

@@ -107,6 +107,37 @@ With that, the Beam version in your environment will be the latest release
 candidate, and you can go ahead and run your tests to verify that everything
 works well.
 
+### Validating Prism Runner RC against RC SDKs
+
+Replace v2.59.0-RC1 with the tag of the RC version being validated.
+
+#### Python
+
+To validate the prism runner with Python,
+`--runner=PrismRunner --prism_location=https://github.com/apache/beam/releases/tag/v2.59.0-RC1 --prism_beam_version_override=v2.59.0`
+
+* The `runner` flag sets Beam to use Prism.
+* The `prism_location` sets the source of Prism assets.
+* The `prism_beam_version_override` flag sets what those artifacts are labeled as.
+* The assets are packaged as the final release version, so the override is required.
+
+#### Java
+
+For Gradle, add the Prism, and the JAMM depdendencies to your `build.gradle`.
+
+```
+ implementation "org.apache.beam:beam-runners-prism-java:2.59.0"
+ implementation "com.github.jbellis:jamm:0.4.0"
+```
+
+Then add the following flags, substituting the version accordingly.
+
+`--runner=PrismRunner --prismLocation="https://github.com/apache/beam/releases/tag/v2.59.0-RC1/" --prismVersionOverride=v2.59.0
+
+* The `runner` flag sets Beam to use Prism.
+* The `prismLocation` sets the source of Prism assets, specifically the zip file of the version in question.
+
+
 ### Configuring a Go build to validate a Beam release candidate
 
 For Go SDK releases, you can fetch the Go SDK RC using [`go get`](https://golang.org/ref/mod#go-get),