From c098372ae55f23eb466a9533cffc5de804a1d9a3 Mon Sep 17 00:00:00 2001 From: Ryan Nett Date: Sat, 26 Jun 2021 19:57:57 -0700 Subject: [PATCH] Working prototype Signed-off-by: Ryan Nett --- tensorflow-core/tensorflow-core-api/pom.xml | 11 ++++ .../annotations/org/tensorflow/op/Ops.java | 6 +- .../tensorflow/internal/c_api/TFLogEntry.java | 2 +- .../tensorflow/internal/c_api/TFLogSink.java | 16 +++++- .../java/org/tensorflow/NativeLogSink.java | 57 +++++++++++++++++++ .../main/java/org/tensorflow/TensorFlow.java | 9 +++ .../internal/c_api/presets/tensorflow.java | 18 +++++- tensorflow-framework/pom.xml | 6 ++ 8 files changed, 116 insertions(+), 9 deletions(-) create mode 100644 tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/NativeLogSink.java diff --git a/tensorflow-core/tensorflow-core-api/pom.xml b/tensorflow-core/tensorflow-core-api/pom.xml index fd96cca67fd..fbbf81e458e 100644 --- a/tensorflow-core/tensorflow-core-api/pom.xml +++ b/tensorflow-core/tensorflow-core-api/pom.xml @@ -47,6 +47,11 @@ ndarray ${ndarray.version} + + org.slf4j + slf4j-api + 1.7.31 + org.junit.jupiter junit-jupiter-api @@ -73,6 +78,12 @@ jmh-generator-annprocess test + + org.slf4j + slf4j-jdk14 + 1.7.31 + test + diff --git a/tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow/op/Ops.java b/tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow/op/Ops.java index 7abb451be2d..c8b7ef1f873 100644 --- a/tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow/op/Ops.java +++ b/tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow/op/Ops.java @@ -376,10 +376,10 @@ public final class Ops { public final SignalOps signal; - public final QuantizationOps quantization; - public final TrainOps train; + public final QuantizationOps quantization; + private final Scope scope; private Ops(Scope scope) { @@ -402,8 +402,8 @@ private Ops(Scope scope) { math = new MathOps(this); audio = new AudioOps(this); signal = new SignalOps(this); - quantization = new QuantizationOps(this); train = new TrainOps(this); + quantization = new QuantizationOps(this); } /** diff --git a/tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/internal/c_api/TFLogEntry.java b/tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/internal/c_api/TFLogEntry.java index 7835d15a8a4..d0afe9368d8 100644 --- a/tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/internal/c_api/TFLogEntry.java +++ b/tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/internal/c_api/TFLogEntry.java @@ -26,5 +26,5 @@ public class TFLogEntry extends Pointer { public native @StdString @Cast({"char*", "std::string&&"}) BytePointer FName(); public native int Line(); public native @StdString @Cast({"char*", "std::string&&"}) BytePointer ToString(); - public native @StdString @Cast({"", "", "std::string"}) BytePointer text_message(); + } diff --git a/tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/internal/c_api/TFLogSink.java b/tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/internal/c_api/TFLogSink.java index 456b6dbbbd8..9e260ebc5d8 100644 --- a/tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/internal/c_api/TFLogSink.java +++ b/tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/internal/c_api/TFLogSink.java @@ -12,8 +12,20 @@ @Namespace("tensorflow") @Properties(inherit = org.tensorflow.internal.c_api.presets.tensorflow.class) public class TFLogSink extends Pointer { static { Loader.load(); } + /** Default native constructor. */ + public TFLogSink() { super((Pointer)null); allocate(); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public TFLogSink(long size) { super((Pointer)null); allocateArray(size); } /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ public TFLogSink(Pointer p) { super(p); } + private native void allocate(); + private native void allocateArray(long size); + @Override public TFLogSink position(long position) { + return (TFLogSink)super.position(position); + } + @Override public TFLogSink getPointer(long i) { + return new TFLogSink(this).position(position + i); + } // `Send` is called synchronously during the log statement. The logging @@ -24,7 +36,7 @@ public class TFLogSink extends Pointer { // `e` is guaranteed to remain valid until the subsequent call to // `WaitTillSent` completes, so implementations may store a pointer to or // copy of `e` (e.g. in a thread local variable) for use in `WaitTillSent`. - public native void Send(@Const @ByRef TFLogEntry entry); + @Virtual(true) public native void Send(@Const @ByRef TFLogEntry entry); // `WaitTillSent` blocks the calling thread (the thread that generated a log // message) until the sink has finished processing the log message. @@ -34,5 +46,5 @@ public class TFLogSink extends Pointer { // The default implementation returns immediately. Like `Send`, // implementations should be careful not to call `LOG` or `CHECK or take any // locks that might be held by the `LOG` caller, to avoid deadlock. - public native void WaitTillSent(); + @Virtual public native void WaitTillSent(); } diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/NativeLogSink.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/NativeLogSink.java new file mode 100644 index 00000000000..41a2f628af6 --- /dev/null +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/NativeLogSink.java @@ -0,0 +1,57 @@ +/* + Copyright 2021 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ======================================================================= + + */ +package org.tensorflow; + +import org.bytedeco.javacpp.Pointer; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.tensorflow.internal.c_api.TFLogEntry; +import org.tensorflow.internal.c_api.TFLogSink; +import org.tensorflow.internal.c_api.global.tensorflow; + +public class NativeLogSink extends TFLogSink { + private static final ILoggerFactory factory = LoggerFactory.getILoggerFactory(); + private static final Logger logger = LoggerFactory.getLogger(NativeLogSink.class); + NativeLogSink() { + super(); + } + + @Override + public void Send(TFLogEntry entry) { + //TODO make work, blocked by https://github.com/tensorflow/tensorflow/issues/44995#issuecomment-869091090 + System.out.printf( + "Log message: Severity: %d, Fname: %s, line: %s, string: %s\n", entry.log_severity(), entry.FName().getString(), entry.Line(), entry.ToString().getString()); +// Logger logger = factory.getLogger(entry.FName().getString()); +// switch (entry.log_severity()){ +// case tensorflow.kWarning: +// logger.warn(entry.ToString().getString()); +// break; +// case tensorflow.kError: +// case tensorflow.kFatal: +// logger.error(entry.ToString().getString()); +// break; +// default: +// logger.info(entry.ToString().getString()); +// } + } + + @Override + public void WaitTillSent() { + } +} diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/TensorFlow.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/TensorFlow.java index 23f4c62bc7f..4f823660d79 100644 --- a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/TensorFlow.java +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/TensorFlow.java @@ -15,6 +15,7 @@ */ package org.tensorflow; +import static org.tensorflow.internal.c_api.global.tensorflow.TFAddLogSink; import static org.tensorflow.internal.c_api.global.tensorflow.TF_DeleteBuffer; import static org.tensorflow.internal.c_api.global.tensorflow.TF_DeleteLibraryHandle; import static org.tensorflow.internal.c_api.global.tensorflow.TF_GetAllOpList; @@ -96,6 +97,13 @@ public static OpList loadLibrary(String filename) { } } + @SuppressWarnings("FieldCanBeLocal") + private static NativeLogSink sink; + private static void setupLogger(){ + sink = new NativeLogSink(); + TFAddLogSink(sink); + } + private static TF_Library libraryLoad(String filename) { try (PointerScope scope = new PointerScope()) { TF_Status status = TF_Status.newStatus(); @@ -137,5 +145,6 @@ private TensorFlow() {} e.printStackTrace(); throw e; } + setupLogger(); } } diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java index c2befde86f5..a2ac124d58b 100644 --- a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java @@ -403,15 +403,27 @@ public void map(InfoMap infoMap) { .cast() .valueTypes("long") .pointerTypes("LongPointer", "long[]")) - .put(new Info("string", "std::string", "tensorflow::string").annotations("@StdString") - .valueTypes("@Cast({\"char*\", \"std::string&&\"}) BytePointer", "@Cast({\"char*\", \"std::string&&\"}) String") - .pointerTypes("@Cast({\"char*\", \"std::string*\"}) BytePointer")) + .put( + new Info("string", "std::string", "tensorflow::string") + .annotations("@StdString") + .valueTypes( + "@Cast({\"char*\", \"std::string&&\"}) BytePointer", + "@Cast({\"char*\", \"std::string&&\"}) String") + .pointerTypes("@Cast({\"char*\", \"std::string*\"}) BytePointer")) +// .put( +// new Info("absl::string_view") +// .annotations("@StdString") +// .valueTypes( +// "@Cast({\"char*\", \"std::string&&\", \"std::string\"}) BytePointer", +// "@Cast({\"char*\", \"std::string&&\", \"std::string\"}) String")) .put( new Info("absl::LogSeverity", "LogSeverity", "tensorflow::LogSeverity") .cast() .valueTypes("int") .pointerTypes("IntPointer", "int[]")) .put(new Info("tensorflow::TFLogEntry").purify()) + .put(new Info("tensorflow::TFLogSink").virtualize()) + .put(new Info("tensorflow::TFLogEntry::text_message").skip()) .put( new Info( "tensorflow::internal::LogEveryNSecState", diff --git a/tensorflow-framework/pom.xml b/tensorflow-framework/pom.xml index af7f47815d5..ac4c24f774d 100644 --- a/tensorflow-framework/pom.xml +++ b/tensorflow-framework/pom.xml @@ -64,6 +64,12 @@ jmh-generator-annprocess test + + org.slf4j + slf4j-jdk14 + 1.7.31 + test + org.tensorflow