[FSTORE-1406] Delete Hive table only if it exists when deleting featu…

…re group (#1796)

* init

* fix null exception if path doesnt exist

* add test

* fix getSchema

* minor improvement

hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb

@@ -384,6 +384,23 @@
  expect_status_details(200)
  end
 
+ it "should be able to delete a cached featuregroup from the featurestore if hive table was already deleted" do
+ project = get_project
+ featurestore_id = get_featurestore_id(project.id)
+ json_result, featuregroup_name = create_cached_featuregroup(project.id, featurestore_id)
+ parsed_json = JSON.parse(json_result)
+ expect_status_details(201)
+
+ # delete feature group offlinefs directory
+ delete "#{ENV['HOPSWORKS_API']}/project/#{project.id.to_s}/dataset//apps/hive/warehouse/#{project.projectname.downcase}_featurestore.db/#{parsed_json["name"]}_#{parsed_json["version"]}"
+ expect_status_details(204)
+
+ featuregroup_id = parsed_json["id"]
+ delete_featuregroup_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + project.id.to_s + "/featurestores/" + featurestore_id.to_s + "/featuregroups/" + featuregroup_id.to_s
+ delete delete_featuregroup_endpoint
+ expect_status_details(200)
+ end
+
  it "should be able to clear the contents of a cached featuregroup in the featurestore" do
  project = get_project
  featurestore_id = get_featurestore_id(project.id)

hopsworks-common/src/main/java/io/hops/hopsworks/common/commands/featurestore/search/SearchFSCommandLogger.java

@@ -64,82 +64,82 @@ public long count() {
 
  public void create(Featuregroup featureGroup) throws FeaturestoreException {
  SearchFSCommand command = getCommand(featureGroup, SearchFSCommandOp.CREATE);
- commandFacade.persistAndFlush(command);
+ persistCommand(command);
  }
 
  public void create(FeatureView featureView) throws FeaturestoreException {
  SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.CREATE);
- commandFacade.persistAndFlush(command);
+ persistCommand(command);
  }
 
  public void create(TrainingDataset trainingDataset) throws FeaturestoreException {
  SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.CREATE);
- commandFacade.persistAndFlush(command);
+ persistCommand(command);
  }
 
  public void updateMetadata(Featuregroup featureGroup) throws FeaturestoreException {
  SearchFSCommand command = getCommand(featureGroup, SearchFSCommandOp.UPDATE_METADATA);
- commandFacade.persistAndFlush(command);
+ persistCommand(command);
  }
 
  public void updateMetadata(FeatureView featureView) throws FeaturestoreException {
  SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.UPDATE_METADATA);
- commandFacade.persistAndFlush(command);
+ persistCommand(command);
  }
 
  public void updateMetadata(TrainingDataset trainingDataset) throws FeaturestoreException {
  SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.UPDATE_METADATA);
- commandFacade.persistAndFlush(command);
+ persistCommand(command);
  }
 
  public void updateKeywords(Featuregroup featureGroup) throws FeaturestoreException {
  SearchFSCommand command = getCommand(featureGroup, SearchFSCommandOp.UPDATE_KEYWORDS);
- commandFacade.persistAndFlush(command);
+ persistCommand(command);
  }
 
  public void updateKeywords(FeatureView featureView) throws FeaturestoreException {
  SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.UPDATE_KEYWORDS);
- commandFacade.persistAndFlush(command);
+ persistCommand(command);
  }
 
  public void updateKeywords(TrainingDataset trainingDataset) throws FeaturestoreException {
  SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.UPDATE_KEYWORDS);
- commandFacade.persistAndFlush(command);
+ persistCommand(command);
  }
 
  public void updateTags(Featuregroup featureGroup) throws FeaturestoreException {
  SearchFSCommand command = getCommand(featureGroup, SearchFSCommandOp.UPDATE_TAGS);
- commandFacade.persistAndFlush(command);
+ persistCommand(command);
  }
 
  public void updateTags(FeatureView featureView) throws FeaturestoreException {
  SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.UPDATE_TAGS);
- commandFacade.persistAndFlush(command);
+ persistCommand(command);
  }
 
  public void updateTags(TrainingDataset trainingDataset) throws FeaturestoreException {
  SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.UPDATE_TAGS);
- commandFacade.persistAndFlush(command);
+ persistCommand(command);
  }
 
  public void delete(Featuregroup featuregroup) throws FeaturestoreException {
  SearchFSCommand command = getCommand(featuregroup, SearchFSCommandOp.DELETE_ARTIFACT);
- commandFacade.persistAndFlush(command);
+ persistCommand(command);
  }
 
  public void delete(FeatureView featureView) throws FeaturestoreException {
  SearchFSCommand command = getCommand(featureView, SearchFSCommandOp.DELETE_ARTIFACT);
- commandFacade.persistAndFlush(command);
+ persistCommand(command);
  }
 
  public void delete(TrainingDataset trainingDataset) throws FeaturestoreException {
  SearchFSCommand command = getCommand(trainingDataset, SearchFSCommandOp.DELETE_ARTIFACT);
- commandFacade.persistAndFlush(command);
+ persistCommand(command);
  }
 
  public void delete(Project project) {
  SearchFSCommand command = getCommand(project, SearchFSCommandOp.DELETE_PROJECT);
- commandFacade.persistAndFlush(command);
+ persistCommand(command);
  }
 
  private SearchFSCommand getCommand(Featuregroup featureGroup, SearchFSCommandOp op) throws FeaturestoreException {
@@ -207,4 +207,10 @@ private SearchFSCommand getCommand(Project p, SearchFSCommandOp op) {
  command.setOp(op);
  return command;
  }
+
+ private void persistCommand(SearchFSCommand command) {
+ if (command != null) {
+ commandFacade.persistAndFlush(command);
+ }
+ }
 }

hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/cached/OfflineFeatureGroupController.java

@@ -33,6 +33,7 @@
 import org.apache.hadoop.hive.metastore.api.AddDefaultConstraintRequest;
 import org.apache.hadoop.hive.metastore.api.DefaultConstraintsRequest;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
 import org.apache.hadoop.hive.metastore.api.SerDeInfo;
 import org.apache.hadoop.hive.metastore.api.SkewedInfo;
@@ -182,6 +183,8 @@ public void alterHiveTableFeatures(Featurestore featurestore, String tableName,
 
  public List<FeatureGroupFeatureDTO> getSchema(Featurestore featurestore, String tableName,
  Project project, Users user) throws FeaturestoreException {
+ List<FeatureGroupFeatureDTO> featureSchema = new ArrayList<>();
+
  String dbName = featurestoreController.getOfflineFeaturestoreDbName(featurestore.getProject());
  ThriftHiveMetastore.Client client = getMetaStoreClient(project, user);
  Table table;
@@ -191,14 +194,22 @@ public List<FeatureGroupFeatureDTO> getSchema(Featurestore featurestore, String
  table = getTable(client, dbName, tableName);
  schema = getFields(client, dbName, tableName);
  defaultConstraints = getDefaultConstraints(client, "hive", dbName, tableName);
+ } catch (FeaturestoreException e) {
+ if (e.getCause() instanceof NoSuchObjectException) {
+ // If the hive table doesn't exist return the feature group without features
+ // (Otherwise the user would not be able to get/delete the broken feature group in the UI/HSFS,
+ // since he would receive 500 responses to GET requests)
+ LOGGER.log(Level.SEVERE, "Feature group Hive table does not exist", e);
+ return featureSchema;
+ }
+ throw e;
  } finally {
  hiveController.finalizeMetastoreOperation(project, user, client);
  }
 
  // Setup a map of constraint values for easy access
  Map<String, String> defaultConstraintsMap = defaultConstraints.stream()
  .collect(Collectors.toMap(SQLDefaultConstraint::getColumn_name, SQLDefaultConstraint::getDefault_value));
- List<FeatureGroupFeatureDTO> featureSchema = new ArrayList<>();
 
  // Partitions are listed first
  for (FieldSchema fieldSchema : table.getPartitionKeys()) {
@@ -295,6 +306,8 @@ public void dropFeatureGroup(String dbName, String tableName, Project project, U
  try {
  client = hiveController.openUserMetastoreClient(project, user);
  client.drop_table(dbName, tableName, true);
+ } catch (NoSuchObjectException e) {
+ LOGGER.log(Level.INFO, "Hive table being deleted does not exist", e);
  } catch (TException e) {
  throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.COULD_NOT_DELETE_FEATUREGROUP, Level.SEVERE,
  "Error dropping feature group in the Hive Metastore: " + e.getMessage(), e.getMessage(), e);