[HWORKS-859] run notebook conversion and git commands in kube job for…

… ee (#1526)

hopsworks-IT/src/test/ruby/spec/git_spec.rb

@@ -369,6 +369,7 @@
  create_branch(@project[:id], repository_id, "test_branch")
  checkout_branch(@project[:id], repository_id, "test_branch")
  make_commit_in_repo(@project, repository_id)
+ #TODO fix this test it is trying to pull a branch that does not exist
  git_pull(@project[:id], repository_id, remote_name="origin", branch_name="test_branch")
  wait_for_git_operation_completed(@project[:id], repository_id, json_body[:id], "Success")
  end

hopsworks-IT/src/test/ruby/spec/spec_helper.rb

@@ -145,7 +145,7 @@
 
 Airborne.configure do |config|
  config.base_url = "https://#{ENV['WEB_HOST']}:#{ENV['WEB_PORT']}"
- config.timeout = 120
+ config.timeout = 240
 end
 
 def try_start_all_services

hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeBeanParam.java

@@ -17,7 +17,7 @@
 package io.hops.hopsworks.api.featurestore.code;
 
 import com.google.common.base.Strings;
-import io.hops.hopsworks.common.jupyter.JupyterController;
+import io.hops.hopsworks.common.jupyter.NotebookConversion;
 import io.swagger.annotations.ApiParam;
 
 import javax.ws.rs.QueryParam;
@@ -45,13 +45,13 @@ public class CodeBeanParam {
 
  @QueryParam("format")
  @ApiParam(value = "ex. format=html", allowableValues = "format=html")
- private JupyterController.NotebookConversion format;
+ private NotebookConversion format;
 
  public CodeBeanParam(
  @QueryParam("sort_by") String sortBy,
  @QueryParam("filter_by") Set<FilterBy> filter,
  @QueryParam("fields") String fields,
- @QueryParam("format") JupyterController.NotebookConversion format) {
+ @QueryParam("format") NotebookConversion format) {
  this.sortBy = sortBy;
  this.sortBySet = getSortBy(sortBy);
  this.filterSet = filter;
@@ -89,7 +89,7 @@ public Set<String> getFieldSet() {
  return fieldSet;
  }
 
- public JupyterController.NotebookConversion getFormat() {
- return format == null ? JupyterController.NotebookConversion.HTML : format;
+ public NotebookConversion getFormat() {
+ return format == null ? NotebookConversion.HTML : format;
  }
 }

hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeBuilder.java

@@ -20,7 +20,7 @@
 import io.hops.hopsworks.common.dao.AbstractFacade;
 import io.hops.hopsworks.common.featurestore.code.CodeController;
 import io.hops.hopsworks.common.featurestore.code.FeaturestoreCodeFacade;
-import io.hops.hopsworks.common.jupyter.JupyterController;
+import io.hops.hopsworks.common.jupyter.NotebookConversion;
 import io.hops.hopsworks.exceptions.FeaturestoreException;
 import io.hops.hopsworks.exceptions.ServiceException;
 import io.hops.hopsworks.persistence.entity.featurestore.Featurestore;
@@ -104,7 +104,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
  Project project, Users user,
  Featuregroup featuregroup,
  FeaturestoreCode featurestoreCode,
- JupyterController.NotebookConversion format) throws FeaturestoreException, ServiceException {
+ NotebookConversion format) throws FeaturestoreException, ServiceException {
  Path fullCodePath = new Path(codeController.getCodeDirFullPath(project, featuregroup),
  featurestoreCode.getFileName());
  CodeDTO dto = new CodeDTO();
@@ -133,7 +133,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
  Project project, Users user,
  TrainingDataset trainingDataset,
  FeaturestoreCode featurestoreCode,
- JupyterController.NotebookConversion format) throws FeaturestoreException, ServiceException {
+ NotebookConversion format) throws FeaturestoreException, ServiceException {
  Path fullCodePath = new Path(codeController.getCodeDirFullPath(project, trainingDataset),
  featurestoreCode.getFileName());
  CodeDTO dto = new CodeDTO();
@@ -156,7 +156,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
 
  public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
  Project project, Users user, Featurestore featurestore, Featuregroup featuregroup,
- JupyterController.NotebookConversion format)
+ NotebookConversion format)
  throws ServiceException, FeaturestoreException {
  CodeDTO dto = new CodeDTO();
  dto.setHref(uri(uriInfo, project, featurestore, featuregroup));
@@ -180,7 +180,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
 
  public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
  Project project, Users user, Featurestore featurestore, TrainingDataset trainingDataset,
- JupyterController.NotebookConversion format)
+ NotebookConversion format)
  throws ServiceException, FeaturestoreException {
  CodeDTO dto = new CodeDTO();
  dto.setHref(uri(uriInfo, project, featurestore, trainingDataset));
@@ -205,7 +205,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
  public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
  Project project, Users user, Featuregroup featuregroup,
  Integer codeId,
- JupyterController.NotebookConversion format)
+ NotebookConversion format)
  throws FeaturestoreException, ServiceException {
  FeaturestoreCode featurestoreCode = codeFacade.findFeaturestoreCodeById(featuregroup, codeId)
  .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.CODE_NOT_FOUND, Level.FINE));
@@ -216,7 +216,7 @@ public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
  public CodeDTO build(UriInfo uriInfo, ResourceRequest resourceRequest,
  Project project, Users user, TrainingDataset trainingDataset,
  Integer codeId,
- JupyterController.NotebookConversion format)
+ NotebookConversion format)
  throws FeaturestoreException, ServiceException {
  FeaturestoreCode featurestoreCode = codeFacade.findFeaturestoreCodeById(trainingDataset, codeId)
  .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.CODE_NOT_FOUND, Level.FINE));

hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeResource.java

@@ -16,17 +16,17 @@
 
 package io.hops.hopsworks.api.featurestore.code;
 
+import io.hops.hopsworks.api.auth.key.ApiKeyRequired;
 import io.hops.hopsworks.api.filter.AllowedProjectRoles;
 import io.hops.hopsworks.api.filter.Audience;
-import io.hops.hopsworks.api.auth.key.ApiKeyRequired;
 import io.hops.hopsworks.api.jwt.JWTHelper;
 import io.hops.hopsworks.api.util.Pagination;
 import io.hops.hopsworks.common.api.ResourceRequest;
 import io.hops.hopsworks.common.featurestore.code.CodeActions;
 import io.hops.hopsworks.common.featurestore.code.CodeController;
 import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController;
 import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetController;
-import io.hops.hopsworks.common.jupyter.JupyterController;
+import io.hops.hopsworks.common.jupyter.NotebookConversion;
 import io.hops.hopsworks.exceptions.DatasetException;
 import io.hops.hopsworks.exceptions.FeaturestoreException;
 import io.hops.hopsworks.exceptions.HopsSecurityException;
@@ -49,15 +49,15 @@
 import javax.enterprise.context.RequestScoped;
 import javax.servlet.http.HttpServletRequest;
 import javax.ws.rs.BeanParam;
-import javax.ws.rs.POST;
 import javax.ws.rs.GET;
+import javax.ws.rs.POST;
 import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
 import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Context;
 import javax.ws.rs.core.SecurityContext;
 import javax.ws.rs.core.UriInfo;
 
@@ -196,12 +196,12 @@ public Response post(@Context UriInfo uriInfo,
  codeDTO.getFeatureGroupCommitId(), codeDTO.getApplicationId(), featuregroup,
  entityId, databricksNotebook, databricksArchive, type);
  dto = codeBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.CODE),
- project, user, featuregroup, featurestoreCode, JupyterController.NotebookConversion.HTML);
+ project, user, featuregroup, featurestoreCode, NotebookConversion.HTML);
  } else {
  FeaturestoreCode featurestoreCode = codeController.registerCode(project, user, codeDTO.getCommitTime(),
  codeDTO.getApplicationId(), trainingDataset, entityId, databricksNotebook, databricksArchive, type);
  dto = codeBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.CODE),
- project, user, trainingDataset, featurestoreCode, JupyterController.NotebookConversion.HTML);
+ project, user, trainingDataset, featurestoreCode, NotebookConversion.HTML);
  }
 
  return Response.ok().entity(dto).build();

hopsworks-api/src/main/java/io/hops/hopsworks/api/git/GitResource.java

@@ -16,9 +16,9 @@
 package io.hops.hopsworks.api.git;
 
 import com.google.common.base.Strings;
+import io.hops.hopsworks.api.auth.key.ApiKeyRequired;
 import io.hops.hopsworks.api.filter.AllowedProjectRoles;
 import io.hops.hopsworks.api.filter.Audience;
-import io.hops.hopsworks.api.auth.key.ApiKeyRequired;
 import io.hops.hopsworks.api.git.branch.BranchBuilder;
 import io.hops.hopsworks.api.git.branch.BranchDTO;
 import io.hops.hopsworks.api.git.execution.ExecutionBeanParam;
@@ -33,22 +33,21 @@
 import io.hops.hopsworks.api.jwt.JWTHelper;
 import io.hops.hopsworks.api.util.Pagination;
 import io.hops.hopsworks.common.api.ResourceRequest;
+import io.hops.hopsworks.common.dao.project.ProjectFacade;
 import io.hops.hopsworks.common.git.BranchCommits;
 import io.hops.hopsworks.common.git.CloneCommandConfiguration;
+import io.hops.hopsworks.common.git.GitBranchAction;
+import io.hops.hopsworks.common.git.GitCommitDTO;
 import io.hops.hopsworks.common.git.GitController;
 import io.hops.hopsworks.common.git.GitRemotesAction;
-import io.hops.hopsworks.common.git.GitBranchAction;
 import io.hops.hopsworks.common.git.GitRepositoryAction;
-import io.hops.hopsworks.common.git.GitCommitDTO;
-import io.hops.hopsworks.common.git.util.GitCommandConfigurationValidator;
 import io.hops.hopsworks.common.git.RepositoryActionCommandConfiguration;
+import io.hops.hopsworks.common.git.util.GitCommandConfigurationValidator;
 import io.hops.hopsworks.exceptions.DatasetException;
-import io.hops.hopsworks.exceptions.UserException;
-import io.hops.hopsworks.persistence.entity.git.GitOpExecution;
-import io.hops.hopsworks.common.dao.project.ProjectFacade;
 import io.hops.hopsworks.exceptions.GitOpException;
 import io.hops.hopsworks.exceptions.HopsSecurityException;
 import io.hops.hopsworks.jwt.annotation.JWTRequired;
+import io.hops.hopsworks.persistence.entity.git.GitOpExecution;
 import io.hops.hopsworks.persistence.entity.git.GitRepository;
 import io.hops.hopsworks.persistence.entity.project.Project;
 import io.hops.hopsworks.persistence.entity.user.Users;
@@ -165,9 +164,10 @@ public Response gitRepository(@PathParam("repositoryId") Integer repositoryId,
  @ApiKeyRequired(acceptedScopes = {ApiScope.GIT}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"})
  public Response clone(CloneCommandConfiguration commandDTO,
  @Context SecurityContext sc,
+ @Context HttpServletRequest req,
  @Context UriInfo uriInfo,
- @BeanParam ExecutionBeanParam executionBeanParam)
- throws GitOpException, HopsSecurityException, IllegalArgumentException, UserException, DatasetException {
+ @BeanParam ExecutionBeanParam executionBeanParam) throws GitOpException, HopsSecurityException,
+ IllegalArgumentException, DatasetException {
  Users hopsworksUser = jWTHelper.getUserPrincipal(sc);
  GitOpExecution execution = gitController.clone(commandDTO, project, hopsworksUser);
  ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXECUTION);

hopsworks-api/src/main/java/io/hops/hopsworks/api/jupyter/JupyterService.java

@@ -47,25 +47,27 @@
 import io.hops.hopsworks.common.dao.jupyter.JupyterSettingsFacade;
 import io.hops.hopsworks.common.dao.jupyter.config.JupyterDTO;
 import io.hops.hopsworks.common.dao.jupyter.config.JupyterFacade;
-import io.hops.hopsworks.common.jupyter.NotebookDTO;
-import io.hops.hopsworks.common.jupyter.JupyterManager;
-import io.hops.hopsworks.common.jupyter.RemoteFSDriverType;
 import io.hops.hopsworks.common.dao.project.ProjectFacade;
 import io.hops.hopsworks.common.hdfs.HdfsUsersController;
 import io.hops.hopsworks.common.hdfs.Utils;
 import io.hops.hopsworks.common.jobs.spark.SparkController;
 import io.hops.hopsworks.common.jupyter.JupyterController;
 import io.hops.hopsworks.common.jupyter.JupyterJWTManager;
+import io.hops.hopsworks.common.jupyter.JupyterManager;
+import io.hops.hopsworks.common.jupyter.NotebookConversion;
+import io.hops.hopsworks.common.jupyter.NotebookDTO;
+import io.hops.hopsworks.common.jupyter.RemoteFSDriverType;
 import io.hops.hopsworks.common.livy.LivyController;
 import io.hops.hopsworks.common.livy.LivyMsg;
+import io.hops.hopsworks.common.system.job.SystemJobStatus;
 import io.hops.hopsworks.common.user.UsersController;
 import io.hops.hopsworks.common.util.Settings;
 import io.hops.hopsworks.exceptions.GenericException;
 import io.hops.hopsworks.exceptions.HopsSecurityException;
-import io.hops.hopsworks.exceptions.ProjectException;
-import io.hops.hopsworks.exceptions.ServiceException;
 import io.hops.hopsworks.exceptions.JobException;
 import io.hops.hopsworks.exceptions.OpenSearchException;
+import io.hops.hopsworks.exceptions.ProjectException;
+import io.hops.hopsworks.exceptions.ServiceException;
 import io.hops.hopsworks.jwt.annotation.JWTRequired;
 import io.hops.hopsworks.persistence.entity.jobs.configuration.spark.SparkJobConfiguration;
 import io.hops.hopsworks.persistence.entity.jupyter.JupyterMode;
@@ -88,14 +90,14 @@
 import javax.validation.constraints.Min;
 import javax.ws.rs.Consumes;
 import javax.ws.rs.DELETE;
+import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.PUT;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
-import javax.ws.rs.DefaultValue;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.GenericEntity;
 import javax.ws.rs.core.MediaType;
@@ -397,15 +399,18 @@ public Response stopNotebookServer(@Context HttpServletRequest req,
  public Response convertIPythonNotebook(@PathParam("path") String path,
  @Context HttpServletRequest req,
  @Context SecurityContext sc) throws ServiceException {
- String ipynbPath = Utils.getProjectPath(this.project.getName()) + "/" + path;
+ if (path.startsWith("/")) {
+ path = path.replaceFirst("/", "");
+ }
+ String ipynbPath = Utils.getProjectPath(this.project.getName()) + path;
  int extensionIndex = ipynbPath.lastIndexOf(".ipynb");
  StringBuilder pathBuilder = new StringBuilder(ipynbPath.substring(0, extensionIndex)).append(".py");
  String pyAppPath = pathBuilder.toString();
  Users user = jWTHelper.getUserPrincipal(sc);
- JupyterController.NotebookConversion conversionType = jupyterController
-  .getNotebookConversionType(ipynbPath, user, this.project);
- jupyterController.convertIPythonNotebook(project, user, ipynbPath, pyAppPath, conversionType);
- return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).build();
+ NotebookConversion conversionType = jupyterController.getNotebookConversionType(ipynbPath, user, this.project);
+ SystemJobStatus status =
+  jupyterController.convertIPythonNotebook(project, user, ipynbPath, pyAppPath, conversionType);
+ return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(status).build();
  }
 
  @POST

hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/git/GitPaths.java

@@ -15,8 +15,6 @@
  */
 package io.hops.hopsworks.common.dao.git;
 
-import java.io.File;
-
 public class GitPaths {
  private String secret;
  private String gitPath;
@@ -28,11 +26,12 @@ public class GitPaths {
 
  public GitPaths(String privateDir, String secretConfig) {
  this.gitPath = privateDir + secretConfig;
- this.logDirPath = gitPath + File.separator + "git_logs";
- this.confDirPath = gitPath + File.separator + "conf";
- this.certificatesDirPath = gitPath + File.separator + "certificates";
- this.runDirPath = gitPath + File.separator + "run";
- this.tokenPath = gitPath + File.separator + "token";
+ String gitPathWithSlash = this.gitPath.endsWith("/") ? this.gitPath : this.gitPath + "/";
+ this.logDirPath = gitPathWithSlash + "git_logs";
+ this.confDirPath = gitPathWithSlash + "conf";
+ this.certificatesDirPath = gitPathWithSlash + "certificates";
+ this.runDirPath = gitPathWithSlash + "run";
+ this.tokenPath = gitPathWithSlash + "token";
  this.secret = secretConfig;
  }
 

hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/git/GitRepositoryRemotesFacade.java

@@ -72,4 +72,11 @@ public void updateRepositoryRemotes(List<GitRepositoryRemote> remotes, GitReposi
  save(remote);
  }
  }
+
+ public GitRepositoryRemote create(GitRepository repository, String name, String url) {
+ GitRepositoryRemote gitRepositoryRemote = new GitRepositoryRemote(repository, name, url);
+ em.persist(gitRepositoryRemote);
+ em.flush();
+ return gitRepositoryRemote;
+ }
 }

hopsworks-common/src/main/java/io/hops/hopsworks/common/dataset/DatasetController.java

@@ -60,8 +60,10 @@
 import io.hops.hopsworks.common.hdfs.command.HdfsCommandExecutionController;
 import io.hops.hopsworks.common.hdfs.inode.InodeController;
 import io.hops.hopsworks.common.jupyter.JupyterController;
+import io.hops.hopsworks.common.jupyter.NotebookConversion;
 import io.hops.hopsworks.common.provenance.core.HopsFSProvenanceController;
 import io.hops.hopsworks.common.provenance.core.dto.ProvTypeDTO;
+import io.hops.hopsworks.common.system.job.SystemJobStatus;
 import io.hops.hopsworks.common.util.HopsUtils;
 import io.hops.hopsworks.common.util.ProjectUtils;
 import io.hops.hopsworks.common.util.Settings;
@@ -625,10 +627,10 @@ public FilePreviewDTO filePreview(Project project, Users user, Path fullPath, Fi
  throw new DatasetException(RESTCodes.DatasetErrorCode.IMAGE_SIZE_INVALID, Level.FINE);
  }
  } else if(fileExtension.equalsIgnoreCase("ipynb")) {
- String html = jupyterController.convertIPythonNotebook(project, user, fullPath.toString(), "''",
-  JupyterController.NotebookConversion.HTML);
+ SystemJobStatus status = jupyterController.convertIPythonNotebook(project, user, fullPath.toString(), "''",
+ NotebookConversion.HTML);
  filePreviewDTO = new FilePreviewDTO(Settings.FILE_PREVIEW_HTML_TYPE, fileExtension.toLowerCase(),
-  html);
+ status.getLog());
  } else {
  try (DataInputStream dis = new DataInputStream(is)) {
  int sizeThreshold = Settings.FILE_PREVIEW_TXT_SIZE_BYTES; //in bytes