0.0.5

README.md

@@ -1,20 +1,27 @@
 # Headless-Browser
 
-A new, headless browser written in Java with latest JavaScript support. Powered by the GraalJS-Engine.
+A new, headless browser written in Java with latest JavaScript support. Powered by the GraalJS-Engine,
+the NodeJs-Engine and Puppeteer.
 
 ```java
 HBrowser hBrowser = new HBrowser();
-HWindow hWindow = hBrowser.openNewWindow().load("https://wikipedia.org");
+try(HWindow hWindow = hBrowser.openWindowAndLoad("https://wikipedia.org")){
+   // Do stuff    
+}
 ```
 
 ## Features
+You can choose from two JavaScript engines: NodeJs and GraalJs.
 
-- [x] Can load pages and partially execute their JavaScript code. Contributions are needed for implementing
-  all [JS Web-APIs](https://developer.mozilla.org/en-US/docs/Web/API), to achieve full JavaScript
-  support. [Click here to see a list of already implemented APIs and how to implement one.](how-to-implement-a-js-web-api.md)
-- [x] Uses Jsoup for editing HTML directly in Java.
-- [x] Uses GraalJS-Engine to execute JavaScript code.
-- [ ] Access to all JS-Web APIs from within Java.
+The NodeJs engine is currently the default and recommended one.
+Puppeteer gets installed automatically along with the latest NodeJs version.
+Currently, only Windows 64x supported (going to change very soon)!
+
+Puppeteer and NodeJs are directly usable in Java.
+
+Creating a new browser completely in Java is ongoing work. Read more below at "Why contribute?".
+
+Jsoup is used to work with HTML in Java.
 
 ## Installation
 

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.osiris.headlessbrowser</groupId>
     <artifactId>Headless-Browser</artifactId>
-    <version>0.0.4</version>
+    <version>0.0.5</version>
     <repositories>
         <repository>
             <id>jitpack</id>

src/main/java/com/osiris/headlessbrowser/NodeContext.java

@@ -10,19 +10,19 @@
 
 import java.io.*;
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Objects;
+import java.nio.file.Files;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
 
 public class NodeContext implements AutoCloseable {
     private final File installationDir = new File(System.getProperty("user.dir") + "/NodeJS-Installation");
-    private File executableFile;
     private final Process process;
     private final AsyncInputStream processInput;
     private final OutputStream processOutput;
-
     private final PrintStream out = System.out;
+    private File executableFile;
+    private File lastJsCodeExecutionResultFile;
 
     public NodeContext() {
         // Download and install NodeJS into current working directory if no installation found
@@ -94,7 +94,7 @@ public NodeContext() {
             // Tried multiple things without success.
             // Update: Node.exe must be started with this flag to get correct I/O: --interactive
             processInput = new AsyncInputStream(process.getInputStream());
-            processInput.listeners.add(line -> out.println("[Node-JS] " + line));
+            processInput.listeners.add(line -> out.println("[" + this + "] " + line));
             new AsyncInputStream(process.getErrorStream()).listeners.add(line -> System.err.println("[Node-JS-ERROR] " + line));
             processOutput = process.getOutputStream();
             out.println(" SUCCESS!");
@@ -117,7 +117,21 @@ public NodeContext() {
         }
 
         try {
-            executeJavaScript("const executeJavaScriptAndGetResult = null;");
+            Thread.sleep(3000);
+            lastJsCodeExecutionResultFile = new File(executableFile.getParentFile() + "/JavaScriptCodeResult.txt");
+            if (!lastJsCodeExecutionResultFile.exists()) lastJsCodeExecutionResultFile.createNewFile();
+            String resultFilePath = lastJsCodeExecutionResultFile.getAbsolutePath().replace("\\", "/"); // To avoid issues with indows file path formats
+            executeJavaScript("var writeResultToJava = function(result) {\n" +
+                    "var fs = require('fs')\n" +
+                    "fs.writeFile('" + resultFilePath + "', result, err => {\n" + // the result var must be defined in the provided jsCode
+                    "  if (err) {\n" +
+                    "    console.error(err)\n" +
+                    "    return\n" +
+                    "  }\n" +
+                    "  //file written successfully\n" +
+                    "})\n" +
+                    "};" +
+                    "console.log('Context initialised!');");
         } catch (Exception e) {
             e.printStackTrace();
         }
@@ -129,23 +143,45 @@ public void close() throws Exception {
     }
 
     public NodeContext writeLine(String line) throws IOException {
-        processOutput.write("\n".getBytes(StandardCharsets.UTF_8)); // To ensure that multi-lined code from before doesn't affect the next lines
-        processOutput.flush();
-        processOutput.write(".break\n".getBytes(StandardCharsets.UTF_8)); // To ensure that multi-lined code from before doesn't affect the next lines
-        processOutput.flush();
-        processOutput.write((line + "\n").getBytes(StandardCharsets.UTF_8));
-        processOutput.flush();
-        processOutput.write(".break\n".getBytes(StandardCharsets.UTF_8)); // To ensure that multi-lined code from before doesn't affect the next lines
-        processOutput.flush();
-        synchronized (out) {
-            if (line.contains("\n")) {
-                out.println("START ===>");
-                out.println("// Writing line(s) to NodeJS context:");
-                out.println(line);
-                out.println("END <===");
-            } else {
+        if (line.contains("\n")) {
+            synchronized (out) {
+                out.println("Writing multiple lines to NodeJS context:");
+                out.println("START >>>>>>>>>");
+            }
+            int lineNumber = 1;
+            String singleLine = null;
+            try (BufferedReader br = new BufferedReader(new StringReader(line))) {
+                while ((singleLine = br.readLine()) != null) {
+                    synchronized (out) {
+                        out.println(lineNumber + "| " + singleLine);
+                        lineNumber++;
+                    }
+                    processOutput.write("\n".getBytes(StandardCharsets.UTF_8)); // To ensure that multi-lined code from before doesn't affect the next lines
+                    processOutput.flush();
+                    processOutput.write(".break\n".getBytes(StandardCharsets.UTF_8)); // To ensure that multi-lined code from before doesn't affect the next lines
+                    processOutput.flush();
+                    processOutput.write((singleLine + "\n").getBytes(StandardCharsets.UTF_8));
+                    processOutput.flush();
+                    processOutput.write(".break\n".getBytes(StandardCharsets.UTF_8)); // To ensure that multi-lined code from before doesn't affect the next lines
+                    processOutput.flush();
+
+                }
+            }
+            synchronized (out) {
+                out.println("END <<<<<<<<<<<");
+            }
+        } else {
+            synchronized (out) {
                 out.println("Writing line to NodeJS context: " + line);
             }
+            processOutput.write("\n".getBytes(StandardCharsets.UTF_8)); // To ensure that multi-lined code from before doesn't affect the next lines
+            processOutput.flush();
+            processOutput.write(".break\n".getBytes(StandardCharsets.UTF_8)); // To ensure that multi-lined code from before doesn't affect the next lines
+            processOutput.flush();
+            processOutput.write((line + "\n").getBytes(StandardCharsets.UTF_8));
+            processOutput.flush();
+            processOutput.write(".break\n".getBytes(StandardCharsets.UTF_8)); // To ensure that multi-lined code from before doesn't affect the next lines
+            processOutput.flush();
         }
         return this;
     }
@@ -154,8 +190,50 @@ public NodeContext writeLine(String line) throws IOException {
      * Executes JavaScript code from the provided {@link String} in the <br>
      * current {@link NodeContext}.
      */
-    public NodeContext executeJavaScript(String jsCode) throws IOException {
-        writeLine(jsCode);
+    public synchronized NodeContext executeJavaScript(String jsCode) {
+        try {
+            if (jsCode.contains("\n")) {
+                synchronized (out) {
+                    out.println("Executing following JS-Code: ");
+                    out.println("JS-CODE START >");
+                    String singleLine = null;
+                    try (BufferedReader br = new BufferedReader(new StringReader(jsCode))) {
+                        while ((singleLine = br.readLine()) != null) {
+                            synchronized (out) {
+                                out.println(singleLine);
+                            }
+                        }
+                    }
+                    out.println("JS-CODE END <");
+                }
+            } else {
+                synchronized (out) {
+                    out.println("Executing following JS-Code: " + jsCode);
+                }
+            }
+
+            AtomicBoolean wasExecuted = new AtomicBoolean();
+            Consumer<String> listener = line -> wasExecuted.set(true);
+            processInput.listeners.add(listener);
+
+            // Writing stuff directly to the process output/NodeJs REPL console somehow is very error-prone.
+            // That's why instead we create a temp file with the js code in it and load it using the .load command.
+            File tmpJs = new File(executableFile.getParentFile() + "/temp" + new Random().nextInt() + ".js");
+            if (!tmpJs.exists()) tmpJs.createNewFile();
+            Files.write(tmpJs.toPath(), jsCode.getBytes(StandardCharsets.UTF_8));
+            executeJavaScript(tmpJs);
+
+            // Wait until we receive a response, like undefined
+            while (!wasExecuted.get()) {
+                Thread.sleep(100);
+            }
+
+            processInput.listeners.remove(listener);
+            tmpJs.delete();
+        } catch (Exception e) {
+            System.err.println("Error during JavaScript execution! Details: ");
+            throw new RuntimeException(e);
+        }
         return this;
     }
 
@@ -164,48 +242,37 @@ public NodeContext executeJavaScript(String jsCode) throws IOException {
      * <pre>
      *     var result = InsertYourFunctionsResultHere;
      * </pre>
+     * That result will get returned to this Java method.
      */
-    public String executeJavaScriptAndGetResult(String jsCode) throws IOException {
-        File resultFile = new File(executableFile.getParentFile() + "/executeJavaScriptAndGetResult.txt");
-        if (!resultFile.exists()) resultFile.createNewFile();
-        String resultFilePath = resultFile.getAbsolutePath();
-
-        executeJavaScript("executeJavaScriptAndGetResult = function() {\n" +
-                jsCode + "\n" +
-                "var fs = require('fs')\n" +
-                "fs.writeFile('" + resultFilePath + "', result, err => {\n" + // the result var must be defined in the provided jsCode
-                "  if (err) {\n" +
-                "    console.error(err)\n" +
-                "    return\n" +
-                "  }\n" +
-                "  //file written successfully\n" +
-                "})\n" +
-                "};" +
-                "executeJavaScriptAndGetResult();" +
-                "executeJavaScriptAndGetResult = null;");
-
-        StringBuilder result = new StringBuilder();
-        String line = null;
-        try (BufferedReader bufferedReader = new BufferedReader(new FileReader(resultFile))) {
-            while ((line = bufferedReader.readLine()) != null) {
-                result.append(line + "\n");
+    public String executeJavaScriptAndGetResult(String jsCode) {
+        try {
+            executeJavaScript(jsCode);
+            executeJavaScript("writeResultToJava(result);\n");
+
+            StringBuilder result = new StringBuilder();
+            String line = null;
+            try (BufferedReader bufferedReader = new BufferedReader(new FileReader(lastJsCodeExecutionResultFile))) {
+                while ((line = bufferedReader.readLine()) != null) {
+                    result.append(line + "\n");
+                }
             }
-        }
 
-        // Clear the files content because we already got what we need
-        try (BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(resultFile))) {
-            bufferedWriter.write("");
+            // Clear the files content because we already got what we need
+            try (BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(lastJsCodeExecutionResultFile))) {
+                bufferedWriter.write("");
+            }
+            return result.toString();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
         }
-
-        return result.toString();
     }
 
     /**
      * Executes JavaScript code from the provided {@link File} in the <br>
      * current {@link NodeContext}.
      */
     public NodeContext executeJavaScript(File jsFile) throws IOException {
-        writeLine(jsFile.getAbsolutePath());
+        writeLine(".load " + jsFile.getAbsolutePath());
         return this;
     }
 
@@ -248,4 +315,27 @@ public Process executeNpmWithArgs(String... args) throws IOException, Interrupte
         return process;
     }
 
+    public File getInstallationDir() {
+        return installationDir;
+    }
+
+    public File getExecutableFile() {
+        return executableFile;
+    }
+
+    public Process getProcess() {
+        return process;
+    }
+
+    public AsyncInputStream getProcessInput() {
+        return processInput;
+    }
+
+    public OutputStream getProcessOutput() {
+        return processOutput;
+    }
+
+    public PrintStream getOut() {
+        return out;
+    }
 }

src/main/java/com/osiris/headlessbrowser/NodeWindow.java

@@ -19,19 +19,13 @@ public class NodeWindow implements AutoCloseable {
     private boolean enableJavaScript;
     private Map<String, String> customHeaders;
 
-    public static void main(String[] args) throws IOException {
-        //new HBrowser().openWindow().executeJS("const hello = 'hi!';").executeJS("console.log(hello);");
-        System.out.println(new HBrowser().openWindow().load("example.com").getDocument().outerHtml());
-    }
-
     public NodeWindow(HBrowser parentBrowser, boolean enableJavaScript, Map<String, String> customHeaders) {
         this.parentBrowser = parentBrowser;
         this.enableJavaScript = enableJavaScript;
         this.customHeaders = customHeaders;
         try {
             jsContext.npmInstall("puppeteer");
-            jsContext.executeJavaScript("" +
-                    "const puppeteer = require('puppeteer');" +
+            jsContext.executeJavaScript("const puppeteer = require('puppeteer');\n" +
                     "const browser = await puppeteer.launch();\n" +
                     "const page = await browser.newPage();\n");
         } catch (Exception e) {
@@ -61,12 +55,17 @@ public NodeWindow load(String url) throws IOException {
         return this;
     }
 
-    public Document getDocument() throws IOException {
+    public Document getDocument() {
         String rawHtml = jsContext.executeJavaScriptAndGetResult("" +
-                "var result = await page.evaluate(() => document.body.innerHTML);");
+                "var result = await page.evaluate(() => document.body.innerHTML);\n");
         return Jsoup.parse(rawHtml);
     }
 
+    public String getTitle() {
+        return jsContext.executeJavaScriptAndGetResult("" +
+                "var result = await page.title();\n" +
+                "");
+    }
 
     /**
      * Executes the provided JavaScript code in the current context. <br>

src/main/java/com/osiris/headlessbrowser/utils/AsyncInputStream.java

@@ -9,9 +9,9 @@
 import java.util.function.Consumer;
 
 public class AsyncInputStream {
-    public List<Consumer<String>> listeners = new CopyOnWriteArrayList<>();
     private final InputStream inputStream;
     private final Thread thread;
+    public List<Consumer<String>> listeners = new CopyOnWriteArrayList<>();
 
     public AsyncInputStream(InputStream inputStream) {
         this.inputStream = inputStream;

src/test/java/com/osiris/headlessbrowser/GraalContextTest.java

@@ -14,10 +14,10 @@ public static void main(String[] args) {
                 .build()) {
             context.getBindings("js").putMember("javaObj", new MyClass());
             boolean valid = context.eval("js",
-                    "    javaObj.id         == 42" +
-                            " && javaObj.text       == '42'" +
-                            " && javaObj.arr[1]     == 42" +
-                            " && javaObj.ret42()    == 42")
+                            "    javaObj.id         == 42" +
+                                    " && javaObj.text       == '42'" +
+                                    " && javaObj.arr[1]     == 42" +
+                                    " && javaObj.ret42()    == 42")
                     .asBoolean();
             context.eval("js", "javaObj.print('HELLO!!!');");
             assert valid == true;