From c360c8233dcbd1c76d6e1ffaf8a7401d3d819c7b Mon Sep 17 00:00:00 2001 From: Garvit Verma Date: Sun, 26 Apr 2020 03:41:02 +0530 Subject: [PATCH 1/4] added icc file related templates --- core/templates.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/core/templates.yml b/core/templates.yml index 8e5d00512..cfd2d8e3d 100644 --- a/core/templates.yml +++ b/core/templates.yml @@ -28,6 +28,11 @@ # keys: # Shotgun Entity keys (CamelCase) + Show: + type: str + shotgun_entity_type: Project + shotgun_field_name: name + filter_by: alphanumeric Sequence: type: str shotgun_entity_type: Sequence @@ -236,6 +241,10 @@ paths: sequence_publish_area_cdl: '@sequence_publish_area/SCRIPTS/nuke' shot_publish_area_cdl: '@shot_publish_area/SCRIPTS/nuke' + project_publish_area_icc: '@project_publish_area/SCRIPTS/adobe' + sequence_publish_area_icc: '@sequence_publish_area/SCRIPTS/adobe' + shot_publish_area_icc: '@shot_publish_area/SCRIPTS/adobe' + ################################# # COMMON PUBLISH paths @@ -562,6 +571,11 @@ paths: colorprocessfiles_slate_logo_file: '@project_publish_area/etc/reviewsubmit_presets/color_process_files_logo.png' colorprocessfiles_render_script_path: '@project_publish_area/etc/reviewsubmit_presets/nuke_batch_color_process_files.py' + # ICC GENERATION + icc_generation_burnin_nuke_file: '@project_publish_area/etc/reviewsubmit_presets/icc_generation.nk' + icc_generation_slate_logo_file: '@project_publish_area/etc/reviewsubmit_presets/icc_generation_logo.png' + icc_generation_render_script_path: '@project_publish_area/etc/reviewsubmit_presets/nuke_batch_icc_generation.py' + # CDL PUBLISHING # TODO: we only support cc publishing currently project_publish_cdl: '@project_publish_area_cdl/grades/grade_v{version}/{name}.cc' @@ -581,6 +595,16 @@ paths: sequence_publish_cube_symlink: '@sequence_publish_area_cdl/grades/grade/{name}.cube' shot_publish_cube_symlink: '@shot_publish_area_cdl/grades/grade/{name}.cube' + # /dd/shows/[getenv DD_SHOW]/[getenv DD_SEQ]/[getenv DD_SHOT]/SHARED/SCRIPTS/adobe/icc/[getenv DD_SHOW]_[getenv DD_SEQ][getenv DD_SHOT]_avid_log.icc + # TODO: we only support avid log icc profiles + project_publish_icc: '@project_publish_area_icc/icc/icc_v{version}/{Show}_avid_log.icc' + sequence_publish_icc: '@sequence_publish_area_icc/icc/icc_v{version}/{Show}_{Sequence}_avid_log.icc' + shot_publish_icc: '@shot_publish_area_icc/icc/icc_v{version}/{Show}_{Sequence}{Shot}_avid_log.icc' + + project_publish_icc_symlink: '@project_publish_area_icc/icc/{Show}_avid_log.icc' + sequence_publish_icc_symlink: '@sequence_publish_area_icc/icc/{Show}_{Sequence}_avid_log.icc' + shot_publish_icc_symlink: '@shot_publish_area_icc/icc/{Show}_{Sequence}{Shot}_avid_log.icc' + # # The aliases section allows you to map one template to another without worrying # about Toolkit complaining about collisions. When an alias is requested, Toolkit From ae98204ec4bb29c789f853f7fe99c7c58e81bdfe Mon Sep 17 00:00:00 2001 From: Garvit Verma Date: Sun, 26 Apr 2020 03:41:51 +0530 Subject: [PATCH 2/4] setup tk-multi-icc_generation app instance --- env/includes/settings/tk-shell.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/env/includes/settings/tk-shell.yml b/env/includes/settings/tk-shell.yml index 996541ebc..2dce92b91 100644 --- a/env/includes/settings/tk-shell.yml +++ b/env/includes/settings/tk-shell.yml @@ -34,5 +34,6 @@ settings.tk-shell: tk-multi-reviewsubmission: "@settings.tk-multi-reviewsubmission" # add the color processing overrides for review submit tk-multi-colorprocessfiles: "@settings.tk-multi-colorprocessfiles" + tk-multi-icc_generation: "@settings.tk-multi-icc_generation" tk-multi-screeningroom: '@settings.tk-multi-screeningroom.rv' location: '@engines.tk-shell.location' From 74cf3b7a83dcad2e9f985fd1b7b85bee4ba736b6 Mon Sep 17 00:00:00 2001 From: Garvit Verma Date: Sun, 26 Apr 2020 03:42:05 +0530 Subject: [PATCH 3/4] setup tk-multi-icc_generation review submit settings --- .../apps/tk-multi-reviewsubmission.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/env/includes/settings/apps/tk-multi-reviewsubmission.yml b/env/includes/settings/apps/tk-multi-reviewsubmission.yml index f6035efb3..c5a4822da 100644 --- a/env/includes/settings/apps/tk-multi-reviewsubmission.yml +++ b/env/includes/settings/apps/tk-multi-reviewsubmission.yml @@ -52,3 +52,21 @@ settings.tk-multi-colorprocessfiles: mov_has_slate: '@settings.tk-multi-colorprocessfiles.mov_has_slate' entity_burnin_sg_fields: '@settings.tk-multi-reviewsubmission.entity_burnin_sg_fields' task_burnin_sg_fields: '@settings.tk-multi-reviewsubmission.task_burnin_sg_fields' + +settings.tk-multi-icc_generation: + movie_width: '@settings.tk-multi-icc_generation.width' + movie_height: '@settings.tk-multi-icc_generation.height' + movie_path_template: '@settings.tk-multi-icc_generation.movie_path_template' + extra_write_nodes_path_info: '@settings.tk-multi-icc_generation.extra_write_nodes_path_info' + sg_version_name_template: '@settings.tk-multi-icc_generation.sg_version_name_template' + slate_logo: 'icc_generation_slate_logo_file' + codec_settings_hook: '{config}/codec_settings.py' + # app location is same as review submit :) + location: "@apps.tk-multi-reviewsubmission.location" + nuke_linux_path: "@path.linux.nuke" + burnin_path: 'icc_generation_burnin_nuke_file' + render_script: 'icc_generation_render_script_path' + preprocess_nuke_hook: '{config}/tk-multi-icc_generation/preprocess_nuke.py' + mov_has_slate: '@settings.tk-multi-icc_generation.mov_has_slate' + entity_burnin_sg_fields: '@settings.tk-multi-reviewsubmission.entity_burnin_sg_fields' + task_burnin_sg_fields: '@settings.tk-multi-reviewsubmission.task_burnin_sg_fields' \ No newline at end of file From d311dc70c8fd8e8050e2de01c0b2c4e9a5f2235e Mon Sep 17 00:00:00 2001 From: Garvit Verma Date: Sun, 26 Apr 2020 03:42:45 +0530 Subject: [PATCH 4/4] modify color process files hook to allow setting with different instances of review submit app --- .../preprocess_nuke.py | 148 ++++++++++++++++++ .../ingest/color_process_files.py | 52 ++++-- .../tk-multi-publish2/ingest/ingest_files.py | 4 - 3 files changed, 185 insertions(+), 19 deletions(-) create mode 100644 hooks/tk-multi-icc_generation/preprocess_nuke.py diff --git a/hooks/tk-multi-icc_generation/preprocess_nuke.py b/hooks/tk-multi-icc_generation/preprocess_nuke.py new file mode 100644 index 000000000..c07badbfe --- /dev/null +++ b/hooks/tk-multi-icc_generation/preprocess_nuke.py @@ -0,0 +1,148 @@ +""" +Hook for doing any preprocessing to the burnin nuke script. +""" +import sgtk +import datetime +import os +import hashlib +import re +import time + +HookBaseClass = sgtk.get_hook_baseclass() + +class PreprocessNuke(HookBaseClass): + + def get_processed_script(self, nuke_script_path, **kwargs): + replace_data = kwargs.get("fields", {}) + + if replace_data.get("path"): + # TODO: publisher.util.get_publish_name() does this much better + replace_data["file_base_name"] = os.path.basename(replace_data["path"]).split('.')[0] + + context = self.parent.context + if context.entity: + sg_entity_type = context.entity["type"] + sg_filters = [["id", "is", context.entity["id"]]] + + sg_fields = self.parent.get_setting('entity_burnin_sg_fields') + replace_data.update(self.parent.shotgun.find_one(sg_entity_type, + filters=sg_filters, + fields=sg_fields)) + if context.task: + sg_fields = self.parent.get_setting('task_burnin_sg_fields') + if 'duration' in sg_fields and 'time_logs_sum' in sg_fields: + sg_time_data = self.parent.shotgun.find('Task', filters=[['entity', 'is', context.entity], + ['step', 'is', context.step]], + fields=['duration', 'time_logs_sum']) + total_duration = 0 + total_time_logged = 0 + for data in sg_time_data: + total_duration += data['duration'] if data['duration'] else 0 + total_time_logged += data['time_logs_sum'] if data['time_logs_sum'] else 0 + if total_duration: total_duration = str(total_duration / (8.0 * 60.0)) + ' day(s)' + if total_time_logged: total_time_logged = str(total_time_logged / (8.0 * 60.0)) + ' day(s)' + + replace_data.update({'duration': total_duration}) + replace_data.update({'time_logs_sum': total_time_logged}) + + sg_fields = list(set(sg_fields) - set(['duration', 'time_logs_sum'])) + + sg_data = self.parent.shotgun.find_one('Task', filters=[['entity', 'is', context.entity], + ['id', 'is', context.task['id']]], + fields=sg_fields) + replace_data.update(sg_data) + + if not replace_data: + # nothing to replace, nothing to do here + return nuke_script_path + + tmp_file_prefix = "colorprocessfiles_tmp_nuke_script" + tmp_file_name = "%s_%s.nk" % (tmp_file_prefix, hashlib.md5(str(time.time())).hexdigest()) + processed_script_path = os.path.join("/var", "tmp", tmp_file_name) + + self.parent.log_debug("Saving nuke script to: {}".format(processed_script_path)) + with open(nuke_script_path, 'r') as source_script_file, open(processed_script_path, + 'w') as tmp_script_file: + nuke_script_text = source_script_file.read() + nuke_script_text = self._replace_vars(nuke_script_text, replace_data) + tmp_script_file.write(nuke_script_text) + + return processed_script_path + + @staticmethod + def remove_html(string): + return re.sub('<.+?>', '', string) + + def _replace_vars(self, attr, data): + """ + Replace the variables in a nuke script + Variables defined by [* *] or \[* *] + """ + # get rid of nukes escape characters (fancy, huh) + attr = attr.replace("\[*", "[*") + + # look for anything with a [* *] pattern + vars = re.compile("\[\*[A-Za-z_ %/:0-9()\-,.\+]+\*\]").findall(attr) + + dt = datetime.datetime.now() + + for var in vars: + var_tmp = var.replace("[*", "").replace("*]", "") + # Replace the date/time variables + if var_tmp.startswith('date '): + date_str = var_tmp.replace('date ', '') + attr = attr.replace(var, dt.strftime(date_str)) + + # Replace the frame number variables + elif (var_tmp.lower() == "numframes" and + data.get('first_frame') != None and data.get('first_frame') != '' and + data.get('last_frame') != None and data.get('last_frame') != ''): + range = str(int(data.get('lf')) - int(data.get('first_frame'))) + attr = attr.replace(var, range) + + # and the increment that may be at the end of the frame number + elif "+" in var_tmp.lower(): + (tmp, num) = var_tmp.split("+") + sum = str(int(data.get(tmp)) + int(num)) + attr = attr.replace(var, sum) + + # make it easier to enter screen coordinates + # that vary with resolution, by normalizing to + # (-0.5 -0.5) to (0.5 0.5) + elif "screenspace" in var_tmp.lower(): + var_tmp = var_tmp.replace("(", " ").replace(",", " ").replace(")", " ") + (key, xString, yString) = var_tmp.split() + xFloat = float(xString) + 0.5 + yFloat = float(yString) + 0.5 + translateString = ( + "{{SHUFFLE_CONSTANT.actual_format.width*(%s) i} {SHUFFLE_CONSTANT.actual_format.height*(%s) i}}" % ( + str(xFloat), str(yFloat))) + attr = attr.replace(var, translateString) + + # TODO: do we handle this differently? + # Replace the showname + # (now resolved in resolveMainProcessVariables) + elif var_tmp == "showname": + attr = attr.replace(var, str(data.get('showname'))) + + # remove knobs that have a [**] value but nothing in data + elif (data.get(var_tmp) == '' or + data.get(var_tmp) == None or + data.get(var_tmp) == "None"): + regexp = ".*\[\*" + regexp += var_tmp + regexp += "\*\].*" + line = re.compile(regexp).findall(attr) + if line != None and len(line) > 0: + if line[0].count('message') > 0: + attr = attr.replace(var, str('""')) + else: + attr = attr.replace(var, "None") + + else: + replaceval = self.remove_html(str(data.get(var_tmp))) + if (replaceval == ""): + replaceval == '""' + attr = attr.replace(var, str(replaceval)) + return attr + # end replaceVars \ No newline at end of file diff --git a/hooks/tk-multi-publish2/ingest/color_process_files.py b/hooks/tk-multi-publish2/ingest/color_process_files.py index 7664dbfbe..e7c961763 100644 --- a/hooks/tk-multi-publish2/ingest/color_process_files.py +++ b/hooks/tk-multi-publish2/ingest/color_process_files.py @@ -33,9 +33,6 @@ def __init__(self, parent, **kwargs): # call base init super(ColorProcessFilesPlugin, self).__init__(parent, **kwargs) - # cache the color process files app, which is an instance of review submission app. - self.__color_process_files_app = self.parent.engine.apps.get("tk-multi-colorprocessfiles") - @property def settings_schema(self): """ @@ -65,7 +62,8 @@ def settings_schema(self): "description": "", "fields": ["context", "version", "[output]", "[name]", "*"], }, - "default_value": {"2K": "{env_name}_2k_image", "Proxy": "{env_name}_proxy_image"}, + "default_value": {"2K": "{env_name}_2k_image", "Proxy": "{env_name}_proxy_image", + None: "ingest_{env_name}_output_render"}, "description": ( "Dictionary of Identifier to Publish Path " "This identifier will be added to publish name and publish type for creating PublishedFile entity." @@ -79,6 +77,13 @@ def settings_schema(self): # make sure this plugin only accepts render sequences. schema["Item Type Filters"]["default_value"] = ["file.*.sequence"] + schema["Review Submit App Name"] = { + "type": "str", + "allows_empty": True, + "default_value": "tk-multi-colorprocessfiles", + "description": "Name of the Review Submit App, to be used by the plugin." + } + return schema def accept(self, task_settings, item): @@ -106,10 +111,6 @@ def accept(self, task_settings, item): accept_data = super(ColorProcessFilesPlugin, self).accept(task_settings, item) - # this plugin shouldn't accept CDL files! Ever! - if item.type == "file.cdl": - accept_data["accepted"] = False - return accept_data def register_publish(self, task_settings, item, path_to_publish): @@ -260,7 +261,8 @@ def validate(self, task_settings, item): if status: - resolved_identifiers = {item.properties.publish_path: None} + # resolved_identifiers = {item.properties.publish_path: None} + resolved_identifiers = dict() # First copy the item's fields fields = copy.copy(item.properties.fields) @@ -268,10 +270,20 @@ def validate(self, task_settings, item): # Update with the fields from the context fields.update(item.context.as_template_fields()) + review_submit_app_name = task_settings["Review Submit App Name"].value + + # color process files app, which is an instance of review submission app. + review_submit_app = self.parent.engine.apps.get(review_submit_app_name) + + if not review_submit_app: + self.logger.error("Review submission App not found with name, %s." % + review_submit_app_name) + return False + # set review_submission app's env/context based on item (ingest) - self.__color_process_files_app.change_context(item.context) + review_submit_app.change_context(item.context) - extra_write_node_mapping = self.__color_process_files_app.resolve_extra_write_nodes(fields) + extra_write_node_mapping = review_submit_app.resolve_extra_write_nodes(fields) # potential processed paths processed_paths = extra_write_node_mapping.values() @@ -353,8 +365,14 @@ def create_published_files(self, task_settings, item): fields.update(item.context.as_template_fields()) self.logger.info("Processing the frames...") + + review_submit_app_name = task_settings["Review Submit App Name"].value + + # color process files app, which is an instance of review submission app. + review_submit_app = self.parent.engine.apps.get(review_submit_app_name) + # run the render hook - pre_processed_paths = self.__color_process_files_app.render(item.properties.path, fields, first_frame, + pre_processed_paths = review_submit_app.render(item.properties.path, fields, first_frame, last_frame, sg_publish_data_list, item.context.task, item.description, item.get_thumbnail_as_path(), @@ -472,14 +490,18 @@ def _get_movie_path(self, task_settings, item): # Update with the fields from the context fields.update(item.context.as_template_fields()) + review_submit_app_name = task_settings["Review Submit App Name"].value + # color process files app, which is an instance of review submission app. + review_submit_app = self.parent.engine.apps.get(review_submit_app_name) + # Movie output width and height - width = self.__color_process_files_app.get_setting("movie_width") - height = self.__color_process_files_app.get_setting("movie_height") + width = review_submit_app.get_setting("movie_width") + height = review_submit_app.get_setting("movie_height") fields["width"] = width fields["height"] = height # Get an output path for the movie. - output_path_template = self.__color_process_files_app.get_template("movie_path_template") + output_path_template = review_submit_app.get_template("movie_path_template") output_path = output_path_template.apply_fields(fields) return output_path diff --git a/hooks/tk-multi-publish2/ingest/ingest_files.py b/hooks/tk-multi-publish2/ingest/ingest_files.py index 5fa3e9881..57b6fdb4d 100644 --- a/hooks/tk-multi-publish2/ingest/ingest_files.py +++ b/hooks/tk-multi-publish2/ingest/ingest_files.py @@ -111,10 +111,6 @@ def accept(self, task_settings, item): accept_data = super(IngestFilesPlugin, self).accept(task_settings, item) - # this plugin shouldn't accept CDL files! Ever! - if item.type == "file.cdl": - accept_data["accepted"] = False - return accept_data def validate(self, task_settings, item):