From 375938a215e338b8d7a05359f913c3a42b35f407 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Tue, 30 Apr 2024 16:38:12 -0700 Subject: [PATCH 1/9] Remove go wrapper (#1268) --- wrappers/go/.gitignore | 2 - wrappers/go/README.md | 34 -------- wrappers/go/build.sh | 23 ----- wrappers/go/main.go | 84 ------------------- wrappers/go/run-tests.sh | 51 ----------- .../Telemetry/EventLogger/.gitignore | 2 - .../Telemetry/EventLogger/EventLogger.cpp | 82 ------------------ .../Telemetry/EventLogger/EventLogger.h | 23 ----- .../Telemetry/EventLogger/EventLogger.swigcxx | 16 ---- .../Telemetry/EventLogger/build.go.linux | 5 -- .../Telemetry/EventLogger/build.go.mac | 5 -- 11 files changed, 327 deletions(-) delete mode 100644 wrappers/go/.gitignore delete mode 100644 wrappers/go/README.md delete mode 100755 wrappers/go/build.sh delete mode 100644 wrappers/go/main.go delete mode 100755 wrappers/go/run-tests.sh delete mode 100644 wrappers/go/src/Microsoft/Telemetry/EventLogger/.gitignore delete mode 100644 wrappers/go/src/Microsoft/Telemetry/EventLogger/EventLogger.cpp delete mode 100644 wrappers/go/src/Microsoft/Telemetry/EventLogger/EventLogger.h delete mode 100644 wrappers/go/src/Microsoft/Telemetry/EventLogger/EventLogger.swigcxx delete mode 100644 wrappers/go/src/Microsoft/Telemetry/EventLogger/build.go.linux delete mode 100644 wrappers/go/src/Microsoft/Telemetry/EventLogger/build.go.mac diff --git a/wrappers/go/.gitignore b/wrappers/go/.gitignore deleted file mode 100644 index ad5d5054b..000000000 --- a/wrappers/go/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -example - diff --git a/wrappers/go/README.md b/wrappers/go/README.md deleted file mode 100644 index a3a20f4b1..000000000 --- a/wrappers/go/README.md +++ /dev/null @@ -1,34 +0,0 @@ -This README describes how to build GoLang wrapper around native C++ Telemetry SDK. - -This not officially supported Go SDK, rather than a sample showing how to route data from Go to 1DS Collector++ - -# Installing 3rd party dependencies - -## Linux - -``` -apt-get install golang swig -``` - -## Mac OS X - -``` -brew install golang -brew install swig -``` - -# Building - -Build script would auto-detect the OS (Linux or Mac) and compile/link Go .exe for that OS - -``` -./build.sh -``` - -# Running - -Requires root: - -``` -./run-tests.sh -``` diff --git a/wrappers/go/build.sh b/wrappers/go/build.sh deleted file mode 100755 index 7df15d8fc..000000000 --- a/wrappers/go/build.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -## Uncomment for custom path to golang -#export PATH=/build/go/bin:$PATH -#export GOROOT=/build/go -export GOPATH=`pwd` - -# Determine OS -OS_NAME=`uname -a` -case "$OS_NAME" in - *Darwin*) OS=mac ;; - *Linux*) OS=linux ;; - *) echo "WARNING: unsupported OS $OS_NAME , exiting.." - return 0 ;; -esac - -# Link to proper build file for OS -MODULE_PATH=`pwd`/src/Microsoft/Telemetry/EventLogger -unlink $MODULE_PATH/build.go -ln -s $MODULE_PATH/build.go.$OS $MODULE_PATH/build.go - -go version -go build -o example diff --git a/wrappers/go/main.go b/wrappers/go/main.go deleted file mode 100644 index 280f04b87..000000000 --- a/wrappers/go/main.go +++ /dev/null @@ -1,84 +0,0 @@ -// -// Copyright (c) Microsoft Corporation. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -package main - -import ( - "fmt" - "strconv" - "time" - "Microsoft/Telemetry/EventLogger" - "runtime" - "os" - "os/signal" - "syscall" - "runtime/debug" -) - -func getuptime() int64 { - return time.Now().UnixNano() / int64(time.Millisecond) -} - -func printMemoryStats() { - var m runtime.MemStats - runtime.ReadMemStats(&m) - fmt.Printf("\tAlloc = %10vK\tTotalAlloc = %10vK\tSys = %10vK\tNumGC = %10v\n", m.Alloc / 1024, m.TotalAlloc / 1024, m.Sys / 1024, m.NumGC) -} - - -/* This code example shows how to register a signal handler routine in GoLang. - * In case if native code experiences an issue, the signal can be trapped here - * and decision made what to do next - restart the process or attempt to recover - * by reinitializing Aria, etc. - */ -func registerSignalHandler() { - go func() { - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGABRT, syscall.SIGTERM) - buf := make([]byte, 1<<20) - for { - <-sigs - stacklen := runtime.Stack(buf, true) - // Print out detailed stack info to facilitate debugging - fmt.Printf("=== received terminate ===\n*** goroutine dump...\n%s\n*** end\n", buf[:stacklen]) - debug.PrintStack() - os.Exit(-1) - } - }() -} - -func main() { - // trap SIGABRT and SIGTERM - registerSignalHandler() - - // 1DSCppSdkTest sandbox key - token := "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx-xxxx" - - fmt.Println("Hello from Microsoft Telemetry-Go!\n") - logger := EventLogger.NewEventLogger() - logger.Init(token) - done := false - seq := 0 - for done != true { - uptime_start := getuptime() - for i := 0 ; i<10000; i++ { - event := EventLogger.NewStringMap() - event.Set("name", "GoLangTest") - event.Set("key1", "value1") - event.Set("key2", "value2") - event.Set("seq", strconv.Itoa(i)) - logger.LogEvent(event) - /* no defer!! */ EventLogger.DeleteStringMap(event) - } - uptime_end := getuptime() - delta := (uptime_end - uptime_start) - fmt.Printf("[%d] sent a batch of 10000 events in %d ms\n", seq, delta) - printMemoryStats() - time.Sleep(time.Second) - seq++ - } - - logger.Done() - -} diff --git a/wrappers/go/run-tests.sh b/wrappers/go/run-tests.sh deleted file mode 100755 index 79200300b..000000000 --- a/wrappers/go/run-tests.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -# -# Clean-up previous results -# -sudo rm /tmp/aria*.log -sudo rm offline* -sudo rm -f heap* - -BIN=./example - -case $1 in - -"") - echo "Running simple test..." - $BIN - ;; - -1) - echo "Running heap check..." - export PPROF_PATH=/usr/local/bin - export HEAPCHECK=normal -# export HEAPPROFILE=/tmp/heapprof -# export LD_PRELOAD="/usr/lib/libtcmalloc.so" - $BIN - ;; - -2) - echo "Running gdb..." - gdb -ex=r --args $BIN - ;; - -3) - echo "Running valgrind..." - valgrind -v $BIN - ;; - -4) - echo "Running valgrind leak check..." - valgrind -v --track-origins=yes --leak-check=full $BIN -# valgrind --vgdb=yes --vgdb-error=0 ./example - ;; - -5) - echo "Running cgroups 500MB memory test..." - cgcreate -g memory:/500MB - echo $(( 500 * 1024 * 1024 )) > /sys/fs/cgroup/memory/500MB/memory.limit_in_bytes - cgexec -g memory:/500MB $BIN - ;; - -esac diff --git a/wrappers/go/src/Microsoft/Telemetry/EventLogger/.gitignore b/wrappers/go/src/Microsoft/Telemetry/EventLogger/.gitignore deleted file mode 100644 index 522abacaa..000000000 --- a/wrappers/go/src/Microsoft/Telemetry/EventLogger/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -build.go - diff --git a/wrappers/go/src/Microsoft/Telemetry/EventLogger/EventLogger.cpp b/wrappers/go/src/Microsoft/Telemetry/EventLogger/EventLogger.cpp deleted file mode 100644 index ad4548744..000000000 --- a/wrappers/go/src/Microsoft/Telemetry/EventLogger/EventLogger.cpp +++ /dev/null @@ -1,82 +0,0 @@ -// -// Copyright (c) Microsoft Corporation. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -#include "EventLogger.h" - -#include "LogManager.hpp" - -#include -#include -#include - -using namespace std; -using namespace MAT; - -LOGMANAGER_INSTANCE - -static ILogger *logger; - -void EventLogger::Init(std::string token) -{ - auto& configuration = LogManager::GetLogConfiguration(); - configuration[CFG_STR_CACHE_FILE_PATH] = "offlinestorage.db"; // ":memory:"; - configuration[CFG_INT_TRACE_LEVEL_MASK] = 0xFFFFFFFF ^ 128; - configuration[CFG_INT_TRACE_LEVEL_MIN] = ACTTraceLevel_Fatal; - configuration[CFG_INT_SDK_MODE] = SdkModeTypes::SdkModeTypes_CS; - configuration[CFG_INT_MAX_TEARDOWN_TIME] = 5; - configuration[CFG_INT_RAM_QUEUE_SIZE] = 32 * 1024 * 1024; // 32 MB heap limit for sqlite3 - configuration[CFG_INT_CACHE_FILE_SIZE] = 16 * 1024 * 1024; // 16 MB storage file limit - - // TODO: move logger from static to private class member - logger = LogManager::Initialize(token); - - // LogManager::SetTransmitProfile(TRANSMITPROFILE_REALTIME); -} - -void EventLogger::Pause() -{ - LogManager::PauseTransmission(); -} - -void EventLogger::Resume() -{ - LogManager::ResumeTransmission(); -} - -void EventLogger::LogEvent(std::map& event) -{ - static long eventCount = 0; - EventProperties props(event["name"]); - for(auto &kv : event) - { - props.SetProperty(kv.first, kv.second); - } - logger->LogEvent(props); - - eventCount++; - -#if 0 /* emulate async SIGABRT after 20000 events */ - if (eventCount > 20000) { - std::thread t([]() { - std::terminate(); - }); - t.detach(); - } -#endif - - // This is required to fix swig projection memleak : - event.clear(); -} - -void EventLogger::Upload() -{ - printf("LogManager::UploadNow\n"); - LogManager::UploadNow(); -} - -void EventLogger::Done() -{ - printf("LogManager::FlushAndTeardown\n"); - LogManager::FlushAndTeardown(); -} diff --git a/wrappers/go/src/Microsoft/Telemetry/EventLogger/EventLogger.h b/wrappers/go/src/Microsoft/Telemetry/EventLogger/EventLogger.h deleted file mode 100644 index e69f2046e..000000000 --- a/wrappers/go/src/Microsoft/Telemetry/EventLogger/EventLogger.h +++ /dev/null @@ -1,23 +0,0 @@ -// -// Copyright (c) Microsoft Corporation. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -#ifndef EventLogger_H -#define EventLogger_H - -#include -#include - -class EventLogger -{ -public: - EventLogger(){}; - void Init(std::string token); - void LogEvent(std::map& event); - void Pause(); - void Resume(); - void Upload(); - void Done(); -}; - -#endif diff --git a/wrappers/go/src/Microsoft/Telemetry/EventLogger/EventLogger.swigcxx b/wrappers/go/src/Microsoft/Telemetry/EventLogger/EventLogger.swigcxx deleted file mode 100644 index 8855abfb5..000000000 --- a/wrappers/go/src/Microsoft/Telemetry/EventLogger/EventLogger.swigcxx +++ /dev/null @@ -1,16 +0,0 @@ -%module eventlogger - -%include -%include "std_string.i" -%include "std_map.i" -%{ - #include "EventLogger.h" -%} - -namespace std { - - %template(StringMap) map; - -} - -%include EventLogger.h diff --git a/wrappers/go/src/Microsoft/Telemetry/EventLogger/build.go.linux b/wrappers/go/src/Microsoft/Telemetry/EventLogger/build.go.linux deleted file mode 100644 index 9b2e2ea40..000000000 --- a/wrappers/go/src/Microsoft/Telemetry/EventLogger/build.go.linux +++ /dev/null @@ -1,5 +0,0 @@ -package EventLogger -// #cgo CFLAGS: -DGENERIC_VALUE=32 -I/usr/local/include/mat -// #cgo CXXFLAGS: -std=c++11 -I/usr/local/include/mat -// #cgo LDFLAGS: -L/usr/local/lib/x86_64-linux-gnu -lmat -lcurl -lz -lsqlite3 -ldl -import "C" diff --git a/wrappers/go/src/Microsoft/Telemetry/EventLogger/build.go.mac b/wrappers/go/src/Microsoft/Telemetry/EventLogger/build.go.mac deleted file mode 100644 index c12ab60be..000000000 --- a/wrappers/go/src/Microsoft/Telemetry/EventLogger/build.go.mac +++ /dev/null @@ -1,5 +0,0 @@ -package EventLogger -// #cgo CFLAGS: -DGENERIC_VALUE=32 -I/usr/local/include/mat -// #cgo CXXFLAGS: -std=c++11 -I/usr/local/include/mat -// #cgo LDFLAGS: -L/usr/local/lib -framework CoreFoundation -framework IOKit -lmat -lcurl -lz -lsqlite3 -ldl -import "C" From 2c08cbe3d0541888334046cc224bc6a1e1b72aa7 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Tue, 7 May 2024 16:29:53 -0700 Subject: [PATCH 2/9] Add all the included languages to CodeQL list (#1270) --- .github/workflows/codeql-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 8f0ffc527..65c598dbc 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -28,7 +28,7 @@ jobs: matrix: # Override automatic language detection by changing the below list # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] - language: ['cpp'] + language: ['cpp', 'javascript', 'python'] # Learn more... # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection From 6bc16275ef213869efde778537db8be44568b362 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 10 May 2024 20:57:18 -0700 Subject: [PATCH 3/9] Remove unmaintained tool OneDSInspector (#1273) --- tools/decoder/CSProtocol_types.cs | 1185 ----------------- tools/decoder/Decoder.cs | 421 ------ tools/decoder/bond.bond | 131 -- tools/decoder/bond_const.bond | 59 - tools/decoder/bond_const_types.cs | 69 - tools/decoder/compile.cmd | 7 - tools/decoder/decoder.projitems | 16 - tools/decoder/decoder.shproj | 15 - tools/fiddler/OneDSInspector/.gitignore | 17 - tools/fiddler/OneDSInspector/AssemblyInfo.cs | 40 - .../fiddler/OneDSInspector/EventInspector.cs | 241 ---- .../fiddler/OneDSInspector/Fiddler.exe.config | 15 - .../OneDSInspector/OneDSInspector.csproj | 96 -- tools/fiddler/OneDSInspector/PayloadViewer.cs | 119 -- .../fiddler/OneDSInspector/PayloadViewer.resx | 120 -- tools/fiddler/OneDSInspector/README.md | 53 +- tools/fiddler/OneDSInspector/app.config | 11 - tools/fiddler/OneDSInspector/packages.config | 7 - tools/fiddler/OneDSInspector/ref/dummy.dll | 0 tools/server/.gitignore | 24 - tools/server/Program.cs | 71 - tools/server/README.md | 21 - tools/server/Startup.cs | 178 --- tools/server/bond/CSProtocol.bond | 299 ----- tools/server/bond/README.md | 4 - tools/server/bond/bond.bond | 0 tools/server/bond/bond_const.bond | 59 - tools/server/bond/compile.cmd | 5 - tools/server/run.cmd | 29 - tools/server/run.sh | 39 - tools/server/server.csproj | 27 - tools/tools.sln | 41 - 32 files changed, 2 insertions(+), 3417 deletions(-) delete mode 100644 tools/decoder/CSProtocol_types.cs delete mode 100644 tools/decoder/Decoder.cs delete mode 100644 tools/decoder/bond.bond delete mode 100644 tools/decoder/bond_const.bond delete mode 100644 tools/decoder/bond_const_types.cs delete mode 100644 tools/decoder/compile.cmd delete mode 100644 tools/decoder/decoder.projitems delete mode 100644 tools/decoder/decoder.shproj delete mode 100644 tools/fiddler/OneDSInspector/.gitignore delete mode 100644 tools/fiddler/OneDSInspector/AssemblyInfo.cs delete mode 100644 tools/fiddler/OneDSInspector/EventInspector.cs delete mode 100644 tools/fiddler/OneDSInspector/Fiddler.exe.config delete mode 100644 tools/fiddler/OneDSInspector/OneDSInspector.csproj delete mode 100644 tools/fiddler/OneDSInspector/PayloadViewer.cs delete mode 100644 tools/fiddler/OneDSInspector/PayloadViewer.resx delete mode 100644 tools/fiddler/OneDSInspector/app.config delete mode 100644 tools/fiddler/OneDSInspector/packages.config delete mode 100644 tools/fiddler/OneDSInspector/ref/dummy.dll delete mode 100644 tools/server/.gitignore delete mode 100644 tools/server/Program.cs delete mode 100644 tools/server/README.md delete mode 100644 tools/server/Startup.cs delete mode 100644 tools/server/bond/CSProtocol.bond delete mode 100644 tools/server/bond/README.md delete mode 100644 tools/server/bond/bond.bond delete mode 100644 tools/server/bond/bond_const.bond delete mode 100644 tools/server/bond/compile.cmd delete mode 100644 tools/server/run.cmd delete mode 100755 tools/server/run.sh delete mode 100644 tools/server/server.csproj delete mode 100644 tools/tools.sln diff --git a/tools/decoder/CSProtocol_types.cs b/tools/decoder/CSProtocol_types.cs deleted file mode 100644 index ab7edbb00..000000000 --- a/tools/decoder/CSProtocol_types.cs +++ /dev/null @@ -1,1185 +0,0 @@ -// -// Copyright (c) Microsoft Corporation. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// - -//------------------------------------------------------------------------------ -// This code was generated by a tool. -// -// Tool : Bond Compiler 0.10.0.0 -// File : CSProtocol_types.cs -// -// Changes to this file may cause incorrect behavior and will be lost when -// the code is regenerated. -// -//------------------------------------------------------------------------------ - - -// suppress "Missing XML comment for publicly visible type or member" -#pragma warning disable 1591 - - -#region ReSharper warnings -// ReSharper disable PartialTypeWithSinglePart -// ReSharper disable RedundantNameQualifier -// ReSharper disable InconsistentNaming -// ReSharper disable CheckNamespace -// ReSharper disable UnusedParameter.Local -// ReSharper disable RedundantUsingDirective -#endregion - -namespace CsProtocol -{ - using System.Collections.Generic; - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Ingest - { - [global::Bond.Id(1), global::Bond.Required] - public long time { get; set; } - - [global::Bond.Id(2), global::Bond.Required] - public string clientIp { get; set; } - - [global::Bond.Id(3)] - public long auth { get; set; } - - [global::Bond.Id(4)] - public long quality { get; set; } - - [global::Bond.Id(5)] - public long uploadTime { get; set; } - - [global::Bond.Id(6)] - public string userAgent { get; set; } - - [global::Bond.Id(7)] - public string client { get; set; } - - public Ingest() - : this("CsProtocol.Ingest", "Ingest") - {} - - protected Ingest(string fullName, string name) - { - clientIp = ""; - userAgent = ""; - client = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class User - { - [global::Bond.Id(1)] - public string id { get; set; } - - [global::Bond.Id(2)] - public string localId { get; set; } - - [global::Bond.Id(3)] - public string authId { get; set; } - - [global::Bond.Id(4)] - public string locale { get; set; } - - public User() - : this("CsProtocol.User", "User") - {} - - protected User(string fullName, string name) - { - id = ""; - localId = ""; - authId = ""; - locale = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Loc - { - [global::Bond.Id(1)] - public string id { get; set; } - - [global::Bond.Id(2)] - public string country { get; set; } - - [global::Bond.Id(3)] - public string timezone { get; set; } - - public Loc() - : this("CsProtocol.Loc", "Loc") - {} - - protected Loc(string fullName, string name) - { - id = ""; - country = ""; - timezone = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Device - { - [global::Bond.Id(1)] - public string id { get; set; } - - [global::Bond.Id(2)] - public string localId { get; set; } - - [global::Bond.Id(3)] - public string authId { get; set; } - - [global::Bond.Id(4)] - public string authSecId { get; set; } - - [global::Bond.Id(5)] - public string deviceClass { get; set; } - - [global::Bond.Id(6)] - public string orgId { get; set; } - - [global::Bond.Id(7)] - public string orgAuthId { get; set; } - - [global::Bond.Id(8)] - public string make { get; set; } - - [global::Bond.Id(9)] - public string model { get; set; } - - [global::Bond.Id(10)] - public string authIdEnt { get; set; } - - public Device() - : this("CsProtocol.Device", "Device") - {} - - protected Device(string fullName, string name) - { - id = ""; - localId = ""; - authId = ""; - authSecId = ""; - deviceClass = ""; - orgId = ""; - orgAuthId = ""; - make = ""; - model = ""; - authIdEnt = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Os - { - [global::Bond.Id(1)] - public string locale { get; set; } - - [global::Bond.Id(2)] - public string expId { get; set; } - - [global::Bond.Id(3)] - public int bootId { get; set; } - - [global::Bond.Id(4)] - public string name { get; set; } - - [global::Bond.Id(5)] - public string ver { get; set; } - - public Os() - : this("CsProtocol.Os", "Os") - {} - - protected Os(string fullName, string name) - { - locale = ""; - expId = ""; - this.name = ""; - ver = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class App - { - [global::Bond.Id(1)] - public string expId { get; set; } - - [global::Bond.Id(2)] - public string userId { get; set; } - - [global::Bond.Id(3)] - public string env { get; set; } - - [global::Bond.Id(4)] - public int asId { get; set; } - - [global::Bond.Id(5)] - public string id { get; set; } - - [global::Bond.Id(6)] - public string ver { get; set; } - - [global::Bond.Id(7)] - public string locale { get; set; } - - [global::Bond.Id(8)] - public string name { get; set; } - - [global::Bond.Id(9)] - public string sesId { get; set; } - - public App() - : this("CsProtocol.App", "App") - {} - - protected App(string fullName, string name) - { - expId = ""; - userId = ""; - env = ""; - id = ""; - ver = ""; - locale = ""; - this.name = ""; - sesId = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Utc - { - [global::Bond.Id(1)] - public string stId { get; set; } - - [global::Bond.Id(2)] - public string aId { get; set; } - - [global::Bond.Id(3)] - public string raId { get; set; } - - [global::Bond.Id(4)] - public string op { get; set; } - - [global::Bond.Id(5)] - public long cat { get; set; } - - [global::Bond.Id(6)] - public long flags { get; set; } - - [global::Bond.Id(7)] - public string sqmId { get; set; } - - [global::Bond.Id(9)] - public string mon { get; set; } - - [global::Bond.Id(10)] - public int cpId { get; set; } - - [global::Bond.Id(11)] - public string bSeq { get; set; } - - [global::Bond.Id(12)] - public string epoch { get; set; } - - [global::Bond.Id(13)] - public long seq { get; set; } - - [global::Bond.Id(14)] - public double popSample { get; set; } - - [global::Bond.Id(15)] - public long eventFlags { get; set; } - - [global::Bond.Id(16)] - public long wsId { get; set; } - - [global::Bond.Id(17)] - public long wcmp { get; set; } - - [global::Bond.Id(18)] - public long wPId { get; set; } - - public Utc() - : this("CsProtocol.Utc", "Utc") - {} - - protected Utc(string fullName, string name) - { - stId = ""; - aId = ""; - raId = ""; - op = ""; - sqmId = ""; - mon = ""; - bSeq = ""; - epoch = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class M365a - { - [global::Bond.Id(1)] - public string enrolledTenantId { get; set; } - - [global::Bond.Id(2)] - public ulong msp { get; set; } - - public M365a() - : this("CsProtocol.M365a", "M365a") - {} - - protected M365a(string fullName, string name) - { - enrolledTenantId = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Xbl - { - [global::Bond.Id(5)] - public Dictionary claims { get; set; } - - [global::Bond.Id(10)] - public string nbf { get; set; } - - [global::Bond.Id(20)] - public string exp { get; set; } - - [global::Bond.Id(30)] - public string sbx { get; set; } - - [global::Bond.Id(40)] - public string dty { get; set; } - - [global::Bond.Id(50)] - public string did { get; set; } - - [global::Bond.Id(60)] - public string xid { get; set; } - - [global::Bond.Id(70)] - public ulong uts { get; set; } - - [global::Bond.Id(80)] - public string pid { get; set; } - - [global::Bond.Id(90)] - public string dvr { get; set; } - - [global::Bond.Id(100)] - public uint tid { get; set; } - - [global::Bond.Id(110)] - public string tvr { get; set; } - - [global::Bond.Id(120)] - public string sty { get; set; } - - [global::Bond.Id(130)] - public string sid { get; set; } - - [global::Bond.Id(140)] - public long eid { get; set; } - - [global::Bond.Id(150)] - public string ip { get; set; } - - public Xbl() - : this("CsProtocol.Xbl", "Xbl") - {} - - protected Xbl(string fullName, string name) - { - claims = new Dictionary(); - nbf = ""; - exp = ""; - sbx = ""; - dty = ""; - did = ""; - xid = ""; - pid = ""; - dvr = ""; - tvr = ""; - sty = ""; - sid = ""; - ip = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Javascript - { - [global::Bond.Id(10)] - public string libVer { get; set; } - - [global::Bond.Id(15)] - public string osName { get; set; } - - [global::Bond.Id(20)] - public string browser { get; set; } - - [global::Bond.Id(21)] - public string browserVersion { get; set; } - - [global::Bond.Id(25)] - public string platform { get; set; } - - [global::Bond.Id(30)] - public string make { get; set; } - - [global::Bond.Id(35)] - public string model { get; set; } - - [global::Bond.Id(40)] - public string screenSize { get; set; } - - [global::Bond.Id(45)] - public string msfpc { get; set; } - - [global::Bond.Id(50)] - public string mc1Id { get; set; } - - [global::Bond.Id(60)] - public ulong mc1Lu { get; set; } - - [global::Bond.Id(70)] - public bool isMc1New { get; set; } - - [global::Bond.Id(80)] - public string ms0 { get; set; } - - [global::Bond.Id(90)] - public string anid { get; set; } - - [global::Bond.Id(100)] - public string a { get; set; } - - [global::Bond.Id(110)] - public string msResearch { get; set; } - - [global::Bond.Id(120)] - public string csrvc { get; set; } - - [global::Bond.Id(130)] - public string rtCell { get; set; } - - [global::Bond.Id(140)] - public string rtEndAction { get; set; } - - [global::Bond.Id(150)] - public string rtPermId { get; set; } - - [global::Bond.Id(160)] - public string r { get; set; } - - [global::Bond.Id(170)] - public string wtFpc { get; set; } - - [global::Bond.Id(180)] - public string omniId { get; set; } - - [global::Bond.Id(190)] - public string gsfxSession { get; set; } - - [global::Bond.Id(200)] - public string domain { get; set; } - - [global::Bond.Id(210), global::Bond.Required] - public bool userConsent { get; set; } - - [global::Bond.Id(220)] - public string browserLang { get; set; } - - [global::Bond.Id(230)] - public string serviceName { get; set; } - - [global::Bond.Id(999)] - public string dnt { get; set; } - - public Javascript() - : this("CsProtocol.Javascript", "Javascript") - {} - - protected Javascript(string fullName, string name) - { - libVer = ""; - osName = ""; - browser = ""; - browserVersion = ""; - platform = ""; - make = ""; - model = ""; - screenSize = ""; - msfpc = ""; - mc1Id = ""; - ms0 = ""; - anid = ""; - a = ""; - msResearch = ""; - csrvc = ""; - rtCell = ""; - rtEndAction = ""; - rtPermId = ""; - r = ""; - wtFpc = ""; - omniId = ""; - gsfxSession = ""; - domain = ""; - browserLang = ""; - serviceName = ""; - dnt = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Protocol - { - [global::Bond.Id(1)] - public int metadataCrc { get; set; } - - [global::Bond.Id(2)] - public List> ticketKeys { get; set; } - - [global::Bond.Id(3)] - public string devMake { get; set; } - - [global::Bond.Id(4)] - public string devModel { get; set; } - - [global::Bond.Id(5)] - public ulong msp { get; set; } - - public Protocol() - : this("CsProtocol.Protocol", "Protocol") - {} - - protected Protocol(string fullName, string name) - { - ticketKeys = new List>(); - devMake = ""; - devModel = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Receipts - { - [global::Bond.Id(1)] - public long originalTime { get; set; } - - [global::Bond.Id(2)] - public long uploadTime { get; set; } - - [global::Bond.Id(3)] - public string originalName { get; set; } - - [global::Bond.Id(4)] - public ulong flags { get; set; } - - public Receipts() - : this("CsProtocol.Receipts", "Receipts") - {} - - protected Receipts(string fullName, string name) - { - originalName = ""; - flags = 0; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Net - { - [global::Bond.Id(1)] - public string provider { get; set; } - - [global::Bond.Id(2)] - public string cost { get; set; } - - [global::Bond.Id(3)] - public string type { get; set; } - - public Net() - : this("CsProtocol.Net", "Net") - {} - - protected Net(string fullName, string name) - { - provider = ""; - cost = ""; - type = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Sdk - { - [global::Bond.Id(1)] - public string ver { get; set; } - - [global::Bond.Id(2)] - public string epoch { get; set; } - - [global::Bond.Id(3)] - public long seq { get; set; } - - [global::Bond.Id(4)] - public string installId { get; set; } - - [global::Bond.Id(5)] - public string libVer { get; set; } - - public Sdk() - : this("CsProtocol.Sdk", "Sdk") - {} - - protected Sdk(string fullName, string name) - { - ver = ""; - epoch = ""; - installId = ""; - libVer = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Cloud - { - [global::Bond.Id(1)] - public string fullEnvName { get; set; } - - [global::Bond.Id(2)] - public string location { get; set; } - - [global::Bond.Id(3)] - public string environment { get; set; } - - [global::Bond.Id(4)] - public string deploymentUnit { get; set; } - - [global::Bond.Id(5)] - public string name { get; set; } - - [global::Bond.Id(6)] - public string roleInstance { get; set; } - - [global::Bond.Id(7)] - public string role { get; set; } - - public Cloud() - : this("CsProtocol.Cloud", "Cloud") - {} - - protected Cloud(string fullName, string name) - { - fullEnvName = ""; - location = ""; - environment = ""; - deploymentUnit = ""; - this.name = ""; - roleInstance = ""; - role = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Service - { - [global::Bond.Id(1)] - public string name { get; set; } - - [global::Bond.Id(2)] - public string role { get; set; } - - [global::Bond.Id(3)] - public string roleVersion { get; set; } - - public Service() - : this("CsProtocol.Service", "Service") - {} - - protected Service(string fullName, string name) - { - this.name = ""; - role = ""; - roleVersion = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Cs - { - [global::Bond.Id(1)] - public string sig { get; set; } - - public Cs() - : this("CsProtocol.Cs", "Cs") - {} - - protected Cs(string fullName, string name) - { - sig = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Mscv - { - [global::Bond.Id(1)] - public string cV { get; set; } - - public Mscv() - : this("CsProtocol.Mscv", "Mscv") - {} - - protected Mscv(string fullName, string name) - { - cV = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class IntWeb - { - [global::Bond.Id(1)] - public string mc1Id { get; set; } - - [global::Bond.Id(2)] - public string msfpc { get; set; } - - [global::Bond.Id(3)] - public string anid { get; set; } - - [global::Bond.Id(4)] - public string serviceName { get; set; } - - [global::Bond.Id(5)] - public Dictionary mscom { get; set; } - - public IntWeb() - : this("CsProtocol.IntWeb", "IntWeb") - {} - - protected IntWeb(string fullName, string name) - { - mc1Id = ""; - msfpc = ""; - anid = ""; - serviceName = ""; - mscom = new Dictionary(); - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class IntService - { - [global::Bond.Id(1)] - public string fullEnvName { get; set; } - - [global::Bond.Id(2)] - public string location { get; set; } - - [global::Bond.Id(3)] - public string environment { get; set; } - - [global::Bond.Id(4)] - public string deploymentUnit { get; set; } - - [global::Bond.Id(5)] - public string name { get; set; } - - public IntService() - : this("CsProtocol.IntService", "IntService") - {} - - protected IntService(string fullName, string name) - { - fullEnvName = ""; - location = ""; - environment = ""; - deploymentUnit = ""; - this.name = ""; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Web - { - [global::Bond.Id(10)] - public string browser { get; set; } - - [global::Bond.Id(20)] - public string browserVer { get; set; } - - [global::Bond.Id(30)] - public string screenRes { get; set; } - - [global::Bond.Id(40)] - public string domain { get; set; } - - [global::Bond.Id(50), global::Bond.Required] - public bool userConsent { get; set; } - - [global::Bond.Id(60)] - public string browserLang { get; set; } - - [global::Bond.Id(70)] - public bool isManual { get; set; } - - public Web() - : this("CsProtocol.Web", "Web") - {} - - protected Web(string fullName, string name) - { - browser = ""; - browserVer = ""; - screenRes = ""; - domain = ""; - browserLang = ""; - } - } - - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public enum ValueKind - { - ValueInt64 = unchecked((int)0), - ValueUInt64 = unchecked((int)1), - ValueInt32 = unchecked((int)2), - ValueUInt32 = unchecked((int)3), - ValueDouble = unchecked((int)4), - ValueString = unchecked((int)5), - ValueBool = unchecked((int)6), - ValueDateTime = unchecked((int)7), - ValueGuid = unchecked((int)8), - ValueArrayInt64 = unchecked((int)9), - ValueArrayUInt64 = unchecked((int)10), - ValueArrayInt32 = unchecked((int)11), - ValueArrayUInt32 = unchecked((int)12), - ValueArrayDouble = unchecked((int)13), - ValueArrayString = unchecked((int)14), - ValueArrayBool = unchecked((int)15), - ValueArrayDateTime = unchecked((int)16), - ValueArrayGuid = unchecked((int)17), - } - - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public enum PIIKind - { - NotSet = unchecked((int)0), - DistinguishedName = unchecked((int)1), - GenericData = unchecked((int)2), - IPV4Address = unchecked((int)3), - IPv6Address = unchecked((int)4), - MailSubject = unchecked((int)5), - PhoneNumber = unchecked((int)6), - QueryString = unchecked((int)7), - SipAddress = unchecked((int)8), - SmtpAddress = unchecked((int)9), - Identity = unchecked((int)10), - Uri = unchecked((int)11), - Fqdn = unchecked((int)12), - IPV4AddressLegacy = unchecked((int)13), - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class PII - { - [global::Bond.Id(1)] - public PIIKind Kind { get; set; } - - public PII() - : this("CsProtocol.PII", "PII") - {} - - protected PII(string fullName, string name) - { - Kind = PIIKind.NotSet; - } - } - - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public enum CustomerContentKind - { - NotSet = unchecked((int)0), - GenericContent = unchecked((int)1), - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class CustomerContent - { - [global::Bond.Id(1)] - public CustomerContentKind Kind { get; set; } - - public CustomerContent() - : this("CsProtocol.CustomerContent", "CustomerContent") - {} - - protected CustomerContent(string fullName, string name) - { - Kind = CustomerContentKind.NotSet; - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Attributes - { - [global::Bond.Id(1)] - public List pii { get; set; } - - [global::Bond.Id(2)] - public List customerContent { get; set; } - - public Attributes() - : this("CsProtocol.Attributes", "Attributes") - {} - - protected Attributes(string fullName, string name) - { - pii = new List(); - customerContent = new List(); - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Value - { - [global::Bond.Id(1)] - public ValueKind type { get; set; } - - [global::Bond.Id(2)] - public List attributes { get; set; } - - [global::Bond.Id(3)] - public string stringValue { get; set; } - - [global::Bond.Id(4)] - public long longValue { get; set; } - - [global::Bond.Id(5)] - public double doubleValue { get; set; } - - [global::Bond.Id(6)] - public List> guidValue { get; set; } - - [global::Bond.Id(10)] - public List> stringArray { get; set; } - - [global::Bond.Id(11)] - public List> longArray { get; set; } - - [global::Bond.Id(12)] - public List> doubleArray { get; set; } - - [global::Bond.Id(13)] - public List>> guidArray { get; set; } - - public Value() - : this("CsProtocol.Value", "Value") - {} - - protected Value(string fullName, string name) - { - type = ValueKind.ValueString; - attributes = new List(); - stringValue = ""; - guidValue = new List>(); - stringArray = new List>(); - longArray = new List>(); - doubleArray = new List>(); - guidArray = new List>>(); - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Data - { - [global::Bond.Id(1)] - public Dictionary properties { get; set; } - - public Data() - : this("CsProtocol.Data", "Data") - {} - - protected Data(string fullName, string name) - { - properties = new Dictionary(); - } - } - - [global::Bond.Schema] - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public partial class Record - { - [global::Bond.Id(1), global::Bond.Required] - public string ver { get; set; } - - [global::Bond.Id(2), global::Bond.Required] - public string name { get; set; } - - [global::Bond.Id(3), global::Bond.Required] - public long time { get; set; } - - [global::Bond.Id(4)] - public double popSample { get; set; } - - [global::Bond.Id(5)] - public string iKey { get; set; } - - [global::Bond.Id(6)] - public long flags { get; set; } - - [global::Bond.Id(7)] - public string cV { get; set; } - - [global::Bond.Id(20)] - public List extIngest { get; set; } - - [global::Bond.Id(21)] - public List extProtocol { get; set; } - - [global::Bond.Id(22)] - public List extUser { get; set; } - - [global::Bond.Id(23)] - public List extDevice { get; set; } - - [global::Bond.Id(24)] - public List extOs { get; set; } - - [global::Bond.Id(25)] - public List extApp { get; set; } - - [global::Bond.Id(26)] - public List extUtc { get; set; } - - [global::Bond.Id(27)] - public List extXbl { get; set; } - - [global::Bond.Id(28)] - public List extJavascript { get; set; } - - [global::Bond.Id(29)] - public List extReceipts { get; set; } - - [global::Bond.Id(31)] - public List extNet { get; set; } - - [global::Bond.Id(32)] - public List extSdk { get; set; } - - [global::Bond.Id(33)] - public List extLoc { get; set; } - - [global::Bond.Id(34)] - public List extCloud { get; set; } - - [global::Bond.Id(35)] - public List extService { get; set; } - - [global::Bond.Id(36)] - public List extCs { get; set; } - - [global::Bond.Id(37)] - public List extM365a { get; set; } - - [global::Bond.Id(41)] - public List ext { get; set; } - - [global::Bond.Id(42)] - public List extMscv { get; set; } - - [global::Bond.Id(43)] - public List extIntWeb { get; set; } - - [global::Bond.Id(44)] - public List extIntService { get; set; } - - [global::Bond.Id(45)] - public List extWeb { get; set; } - - [global::Bond.Id(51)] - public Dictionary tags { get; set; } - - [global::Bond.Id(60)] - public string baseType { get; set; } - - [global::Bond.Id(61)] - public List baseData { get; set; } - - [global::Bond.Id(70)] - public List data { get; set; } - - public Record() - : this("CsProtocol.Record", "Record") - {} - - protected Record(string fullName, string name) - { - ver = ""; - this.name = ""; - popSample = 100.0; - iKey = ""; - cV = ""; - extIngest = new List(); - extProtocol = new List(); - extUser = new List(); - extDevice = new List(); - extOs = new List(); - extApp = new List(); - extUtc = new List(); - extXbl = new List(); - extJavascript = new List(); - extReceipts = new List(); - extNet = new List(); - extSdk = new List(); - extLoc = new List(); - extCloud = new List(); - extService = new List(); - extCs = new List(); - extM365a = new List(); - ext = new List(); - extMscv = new List(); - extIntWeb = new List(); - extIntService = new List(); - extWeb = new List(); - tags = new Dictionary(); - baseType = ""; - baseData = new List(); - data = new List(); - } - } -} // CsProtocol diff --git a/tools/decoder/Decoder.cs b/tools/decoder/Decoder.cs deleted file mode 100644 index 83d2f9f64..000000000 --- a/tools/decoder/Decoder.cs +++ /dev/null @@ -1,421 +0,0 @@ -// -// Copyright (c) Microsoft Corporation. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// - -// System -using System; -using System.Text; -using System.IO; -using System.IO.Compression; -using System.Collections.Generic; - -// Protocol -using Bond; -using Bond.Protocols; -using Bond.IO.Safe; - -#if NETCOREAPP -// Extensions -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Logging; -#endif - -// JSON parser -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using CsProtocol; -using System.Linq; - -namespace CommonSchema -{ - public class Decoder - { - -#if NETCOREAPP - // Logger passed down from the caller - public ILogger Logger { get; set; } -#endif - - // Raw HTTP post body in input format - private byte[] SavedBody; - - // Decoded output in JSON format - private string DecodedJson; - - /// - /// Initialize Decoder input parameters: request headers and request body. - /// - /// Request headers - /// Request body - public Decoder(IDictionary requestHeaders, byte[] requestBody) - { - if (requestHeaders is null) - { - throw new ArgumentNullException(nameof(requestHeaders)); - } - ClientId = requestHeaders["Client-Id"]; - ContentType = requestHeaders["Content-Type"]; - ContentEncoding = requestHeaders.ContainsKey("Content-Encoding") ? requestHeaders["Content-Encoding"] : ""; - RequestBody = requestBody ?? throw new ArgumentNullException(nameof(requestBody)); - } - - /// - /// Gets or sets the request body. - /// - public byte[] RequestBody - { - get => Encoding.UTF8.GetBytes(DecodedJson); - set => SavedBody = value; - } - - /// - /// Gets or sets the "Client-Id" of the inspected content. - /// - public string ClientId { get; set; } - - /// - /// Gets or sets the "Content-Type" of the inspected content. - /// - public string ContentType { get; set; } - - /// - /// Gets or sets the "Content-Encoding" of the inspected content. - /// - public string ContentEncoding { get; set; } - - /// - /// Indent JSON using tab size provided. - /// - /// JSON string - /// Tab size. Default: 2 - /// - private static string IndentJson(string json, int TabSize = 2) - { - StringReader stringReader = new StringReader(json); - StringWriter stringWriter = new StringWriter(); - JsonTextReader jsonReader = new JsonTextReader(stringReader); - using (JsonTextWriter jsonWriter = new JsonTextWriter(stringWriter) { Formatting = Formatting.Indented, IndentChar = ' ', Indentation = TabSize }) - { - jsonWriter.WriteToken(jsonReader); - } - return stringWriter.ToString(); - } - - private static readonly Dictionary nodeMap = new Dictionary - { - { "extIngest", "injest" }, - { "extProtocol", "protocol" }, - { "extUser", "user" }, - { "extDevice", "device" }, - { "extOs", "os" }, - { "extApp", "app" }, - { "extUtc", "utc" }, - { "extXbl", "xbl" }, - { "extJavascript", "js" }, - { "extReceipts", "receipts" }, - { "extNet", "net" }, - { "extSdk", "sdk" }, - { "extLoc", "loc" }, - { "extCloud", "cloud" }, - { "extService", "service" }, - { "extCs", "cs" }, - { "extM365a", "M365a" }, - { "extMscv", "mscv" }, - { "extIntWeb", "intWeb" }, - { "extIntService", "intService" }, - { "extWeb", "web" } - }; - - /// - /// Convert Bond-style Value object to more readable JSON notation - /// - /// - /// - JToken ToJsonValue(JToken token) - { - JObject jObj = (token.Type == JTokenType.Object) ? (JObject)token: new JObject(); - - // Ref. CSProtocol_types.cs - if (jObj.ContainsKey("type")) - { - ValueKind kind = (CsProtocol.ValueKind)(int)jObj["type"]; - switch (kind) - { - case ValueKind.ValueInt64: - return jObj["longValue"]; - case ValueKind.ValueDouble: - return jObj["doubleValue"]; - case ValueKind.ValueGuid: - return jObj["guidValue"][0]; - case ValueKind.ValueBool: - return new JValue(jObj.Count != 1); - default: - break; - } - } - - if (jObj.ContainsKey("stringValue") && (jObj.Count == 1)) - { - return jObj["stringValue"]; - } - - // TODO: add nicer formatting for non-standard complex types: - // - stringArray - // - longArray - // - doubleArray - // - guidArray - // Currently these are still expressed based on JSON generated from Bond schema. - - return jObj; - } - - /// - /// Pull Common Schema extensions to 2nd-level JSON extension under 'ext' - /// e.g. - /// from: "extNet" : [{"cost":x, "type":y}] - /// to: "ext":{"net":{"cost":x,"type":y}} - /// - /// - /// - /// - private string ToCompactExt(string json) - { - JObject jObj = JObject.Parse(json); - if (!jObj.ContainsKey("ext")) - { - jObj.Add("ext", new JObject()); - } - JObject ext = (JObject)jObj["ext"]; - foreach(KeyValuePair entry in nodeMap) - { - if (jObj.TryGetValue(entry.Key, StringComparison.InvariantCultureIgnoreCase, out JToken parent)) - { - try - { - if (parent.Type == JTokenType.Array) - { - JArray extArr = (JArray)parent; - if (extArr.Count>0) - { - JObject child = (JObject)(extArr.First); - if (child.Count > 0) - { - child = (JObject)(child.DeepClone()); - if (!ext.ContainsKey(entry.Value)) - { - ext.Add(entry.Value, child); - } - } - } - jObj.Remove(entry.Key); - } - } - catch (Exception ex) - { -#if NETCOREAPP - Logger.LogCritical( - "Exception: {ex}\nInvalid JSON tree node: {json}", - ex, - parent.ToString()); -#else - throw; -#endif - } - } - } - - // TODO: - // - consider moving ext.loc.timezone to ext.loc.tz - - // Convert Bond-style event into JSON - if (jObj.TryGetValue("data", out JToken oldData)) - { - var props = oldData[0]["properties"].DeepClone(); - jObj.Remove("data"); - var newData = new JObject(); - for(int i=0; i - /// Indent decoded JSON using TabSize provided. - /// - /// - /// - public string IndentJson(int TabSize = 2) - { - DecodedJson = IndentJson(DecodedJson, TabSize); - return DecodedJson; - } - - /// - /// Transform input HTTP request body to JSON string. - /// - /// Produce compact JSON. Default: true - /// If TabSize is specified, then JSON is pretty-formatted using TabSize - /// - public string ToJson(bool outputCompactJson = true, bool flatExt = true, int TabSize = -1) - { - FromBondToJSONList(outputCompactJson, flatExt); - if (TabSize != -1) - { - IndentJson(TabSize); - } - return DecodedJson; - } - - /// - /// Deserialize the byte data into a DataPackage, then to JSON format, - /// returning list of JSON records. This method also updates the - /// DecodedJson property with JSON containing an array of all records. - /// - /// Whether the output JSON should be - /// compact or human-readble (tabulated). - /// Whether to move ext${X} records under ext - /// for smaller more concise representation similar to JSON-CS protocol. - /// - /// The JSON representation of the data bytes - public List FromBondToJSONList(bool outputCompactJson = true, bool flatExt = true) - { - byte[] data = SavedBody; - - // Before deserializing the payload, check the content encoding first to see if it is - // compressed. - List jsonList = new List(); - - if (!string.IsNullOrEmpty(this.ContentEncoding)) - { - if (this.ContentEncoding == "gzip") - { - data = Gunzip(data); - } - else if (this.ContentEncoding == "deflate") - { - data = Inflate(data); - } - } - - // read using Common Schema protocol - using (MemoryStream outputBuffer = new MemoryStream()) - { - try - { - InputBuffer input = new InputBuffer(data); - CompactBinaryReader reader = new CompactBinaryReader(input); - SimpleJsonWriter writer = new SimpleJsonWriter(outputBuffer); - do - { - Record evt = Deserialize.From(reader); - // exception is thrown here if request is invalid - Serialize.To(writer, evt); - writer.Flush(); - string j = Encoding.UTF8.GetString(outputBuffer.ToArray()); - j = ToCompactExt(j); - jsonList.Add(j); - outputBuffer.SetLength(0); - } while (true); - } - catch (EndOfStreamException) - { -#if NETCOREAPP - Logger.LogDebug("End of Binary Stream"); -#endif - } - catch (Exception ex) - { - // end of input -#if NETCOREAPP - Logger.LogCritical( "Exception: {ex}", ex); -#endif - } - - // flatten all into one buffer representing JSON array of records - outputBuffer.WriteByte((byte)'['); - int i = 0; - jsonList.ForEach((s) => - { - if (i > 0) - outputBuffer.WriteByte((byte)','); - i++; - var buf = Encoding.UTF8.GetBytes(s); - outputBuffer.Write(buf, 0, buf.Length); - }); - outputBuffer.WriteByte((byte)']'); - outputBuffer.WriteByte((byte)'\n'); - - // get as string - string json = Encoding.UTF8.GetString(outputBuffer.ToArray()); - try - { - // pretty-print if necessary - if (!outputCompactJson) - { - json = JToken.Parse(json).ToString(Formatting.Indented); - } - } - catch (Exception) - { - // ignore invalid JSON contents here - } - - // Save result - DecodedJson = json; - } - - // Return list of individual records - return jsonList; - } - - /// - /// Helper routine to gunzip compressed data. - /// - /// The compressed data - /// The gunzip'd data - public static byte[] Gunzip(byte[] data) - { - if (data is null) - { - throw new ArgumentNullException(nameof(data)); - } - - MemoryStream inStream = new MemoryStream(data); - GZipStream gZipStream = new GZipStream(inStream, CompressionMode.Decompress); - MemoryStream memoryStream = new MemoryStream(); - gZipStream.CopyTo(memoryStream); - gZipStream.Dispose(); - return memoryStream.ToArray(); - } - - /// - /// Helper routine to deflate compressed data. - /// - /// The compressed data - /// The delated data - public static byte[] Inflate(byte[] data) - { - if (data is null) - { - throw new ArgumentNullException(nameof(data)); - } - - MemoryStream inStream = new MemoryStream(data); - DeflateStream deflateStream = new DeflateStream(inStream, CompressionMode.Decompress); - MemoryStream outStream = new MemoryStream(); - deflateStream.CopyTo(outStream); - deflateStream.Dispose(); - return outStream.ToArray(); - } - - } - -} diff --git a/tools/decoder/bond.bond b/tools/decoder/bond.bond deleted file mode 100644 index 8c8384316..000000000 --- a/tools/decoder/bond.bond +++ /dev/null @@ -1,131 +0,0 @@ -import "bond_const.bond" - -namespace bond - -struct SerializableExceptionBase -{ - 8189: string message; -} - - -// Empty schema with no fields -struct Void -{ -} - -// Schema with the same memory layout as Windows GUID -struct GUID -{ - 0: uint32 Data1; - 1: uint16 Data2; - 2: uint16 Data3; - 3: uint64 Data4; -} - -// Field modifier enumerator -enum Modifier -{ - Optional, - Required, - RequiredOptional -} - -// Schema used to represent field's default value -struct Variant -{ - 0: uint64 uint_value; - 1: int64 int_value; - 2: double double_value; - 3: string string_value; - 4: wstring wstring_value; - 5: bool nothing; -} - -// Schema representing field or struct metadata -struct Metadata -{ - // Name of the field or struct - 0: string name; - - // Fully qualified name, used only for structs - 1: string qualified_name; - - // Attributes - 2: map attributes; - - // Field modifier, not used for structs - 3: Modifier modifier = Optional; - - // Default value of the field, not used for structs - 4: Variant default_value; -} - -// Schema representing a type within SchemaDef -struct TypeDef -{ - // Type identifier - 0: BondDataType id = BT_STRUCT; - - // Index of struct definition in SchemaDef.structs when id == BT_STRUCT - 1: uint16 struct_def = 0; - - // Type definition for: - // list elements (id == BT_LIST), - // set elements (id == BT_SET), - // or mapped value (id == BT_MAP) - 2: nullable element; - - // Type definition for map key when id == BT_MAP - 3: nullable key; - - // True if the type is bonded; used only when id == BT_STRUCT - 4: bool bonded_type; - - // Distinguish between kinds of lists; used only when id == BT_LIST - // Currently not present in TypeDef, as its addition is breaking some - // users that have already serialized SchemaDef structs. - // 5: ListSubType list_sub_type = NO_SUBTYPE; -} - -// Schema representing a field definition -struct FieldDef -{ - // Field metadata - 0: Metadata metadata; - - // Field ordinal - 1: uint16 id; - - // Field type definition - 2: TypeDef type; -} - -// Schema representing a struct definition -struct StructDef -{ - // Struct metadata - 0: Metadata metadata; - - // Type definition of base struct - 1: nullable base_def; - - // List of field definitions - 2: vector fields; -} - - -// Schema used to represent schema definition -struct SchemaDef -{ - // List of struct definitions referenced in the schema - 0: vector structs; - - // Root struct of the schema - 1: TypeDef root; -} - -// Generic schema with one field of specified type -struct Box -{ - 0: T value; -} diff --git a/tools/decoder/bond_const.bond b/tools/decoder/bond_const.bond deleted file mode 100644 index 4d7c54eef..000000000 --- a/tools/decoder/bond_const.bond +++ /dev/null @@ -1,59 +0,0 @@ -namespace bond - -// Enumerator of Bond meta-schema types -enum BondDataType -{ - BT_STOP = 0, - BT_STOP_BASE = 1, - BT_BOOL = 2, - BT_UINT8 = 3, - BT_UINT16 = 4, - BT_UINT32 = 5, - BT_UINT64 = 6, - BT_FLOAT = 7, - BT_DOUBLE = 8, - BT_STRING = 9, - BT_STRUCT = 10, - BT_LIST = 11, - BT_SET = 12, - BT_MAP = 13, - BT_INT8 = 14, - BT_INT16 = 15, - BT_INT32 = 16, - BT_INT64 = 17, - BT_WSTRING = 18, - BT_UNAVAILABLE= 127 -} - -// Enumerator to distinguish the different subtypes treated as BT_LIST -enum ListSubType -{ - // Used when the type is a list/vector. - // Also used when the subtype cannot be determined. - NO_SUBTYPE = 0, - - // Used when the type is nullable. - NULLABLE_SUBTYPE = 1, - - // Used when the type is a blob. - BLOB_SUBTYPE = 2 -} - -// Magic numbers of predefined protocols -enum ProtocolType -{ - // Actual protocol type is marshaled with payload - MARSHALED_PROTOCOL = 0, - - // Fast binary protocol - FAST_PROTOCOL = 0x464d, - - // Compact binary protocol - COMPACT_PROTOCOL = 0x4243, - - // Simple JSON protocol - SIMPLE_JSON_PROTOCOL = 0x4a53, - - // Simple binary protocol - SIMPLE_PROTOCOL = 0x5053, -} diff --git a/tools/decoder/bond_const_types.cs b/tools/decoder/bond_const_types.cs deleted file mode 100644 index 140c3383c..000000000 --- a/tools/decoder/bond_const_types.cs +++ /dev/null @@ -1,69 +0,0 @@ -// -// Copyright (c) Microsoft Corporation. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// - -//------------------------------------------------------------------------------ -// This code was generated by a tool. -// -// Tool : Bond Compiler 0.10.0.0 -// File : bond_const_types.cs -// -// Changes to this file may cause incorrect behavior and will be lost when -// the code is regenerated. -// -//------------------------------------------------------------------------------ - - -// suppress "Missing XML comment for publicly visible type or member" -#pragma warning disable 1591 - - -#region ReSharper warnings -// ReSharper disable PartialTypeWithSinglePart -// ReSharper disable RedundantNameQualifier -// ReSharper disable InconsistentNaming -// ReSharper disable CheckNamespace -// ReSharper disable UnusedParameter.Local -// ReSharper disable RedundantUsingDirective -#endregion - -namespace bond -{ - using System.Collections.Generic; - - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public enum BondDataType - { - BT_STOP = unchecked((int)0), - BT_STOP_BASE = unchecked((int)1), - BT_BOOL = unchecked((int)2), - BT_UINT8 = unchecked((int)3), - BT_UINT16 = unchecked((int)4), - BT_UINT32 = unchecked((int)5), - BT_UINT64 = unchecked((int)6), - BT_FLOAT = unchecked((int)7), - BT_DOUBLE = unchecked((int)8), - BT_STRING = unchecked((int)9), - BT_STRUCT = unchecked((int)10), - BT_LIST = unchecked((int)11), - BT_SET = unchecked((int)12), - BT_MAP = unchecked((int)13), - BT_INT8 = unchecked((int)14), - BT_INT16 = unchecked((int)15), - BT_INT32 = unchecked((int)16), - BT_INT64 = unchecked((int)17), - BT_WSTRING = unchecked((int)18), - BT_UNAVAILABLE = unchecked((int)127), - } - - [System.CodeDom.Compiler.GeneratedCode("gbc", "0.10.0.0")] - public enum ProtocolType - { - MARSHALED_PROTOCOL = unchecked((int)0), - FAST_PROTOCOL = unchecked((int)17997), - COMPACT_PROTOCOL = unchecked((int)16963), - SIMPLE_JSON_PROTOCOL = unchecked((int)19027), - SIMPLE_PROTOCOL = unchecked((int)20563), - } -} // bond diff --git a/tools/decoder/compile.cmd b/tools/decoder/compile.cmd deleted file mode 100644 index 498de3f91..000000000 --- a/tools/decoder/compile.cmd +++ /dev/null @@ -1,7 +0,0 @@ -setlocal -set "PATH=%~dp0\..\;%PATH%" -pushd "%~dp0" -copy /Y %~dp0\..\..\lib\bond\CsProtocol.bond %~dp0 -gbc.exe c# CSProtocol.bond -del CSProtocol.bond -popd diff --git a/tools/decoder/decoder.projitems b/tools/decoder/decoder.projitems deleted file mode 100644 index 475c6965b..000000000 --- a/tools/decoder/decoder.projitems +++ /dev/null @@ -1,16 +0,0 @@ - - - - $(MSBuildAllProjects);$(MSBuildThisFileFullPath) - true - 777cf875-abf2-4202-84c5-9e8161487fbb - - - decoder - - - - - - - \ No newline at end of file diff --git a/tools/decoder/decoder.shproj b/tools/decoder/decoder.shproj deleted file mode 100644 index 56c5f7278..000000000 --- a/tools/decoder/decoder.shproj +++ /dev/null @@ -1,15 +0,0 @@ - - - - - {6A206BE3-9975-4952-8E1D-35BC2F47431C} - 14.0 - - - - - - - - - \ No newline at end of file diff --git a/tools/fiddler/OneDSInspector/.gitignore b/tools/fiddler/OneDSInspector/.gitignore deleted file mode 100644 index bf87ad98e..000000000 --- a/tools/fiddler/OneDSInspector/.gitignore +++ /dev/null @@ -1,17 +0,0 @@ -# Don't self ignore -!.gitignore -!.tfignore - -# Default project build output dirs -bin/ -obj/ -csx/ - -# Miscellaneous files to ignore -*.suo -*.user -*.sdf -*.opensdf -*.backup.* -*.DotSettings -*\rcf diff --git a/tools/fiddler/OneDSInspector/AssemblyInfo.cs b/tools/fiddler/OneDSInspector/AssemblyInfo.cs deleted file mode 100644 index d11802ee9..000000000 --- a/tools/fiddler/OneDSInspector/AssemblyInfo.cs +++ /dev/null @@ -1,40 +0,0 @@ -// -// Copyright (c) Microsoft Corporation. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// - -using System.Reflection; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("OneDSInspector")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("Microsoft Corporation")] -[assembly: AssemblyProduct("OneDSInspector")] -[assembly: AssemblyCopyright("Copyright (c) Microsoft Corporation. All rights reserved.")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("b1916b1c-9108-4965-8d22-ad840cf2c069")] - -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("3.4.0.0")] -[assembly: AssemblyFileVersion("3.4.0.0")] diff --git a/tools/fiddler/OneDSInspector/EventInspector.cs b/tools/fiddler/OneDSInspector/EventInspector.cs deleted file mode 100644 index 757abf8bd..000000000 --- a/tools/fiddler/OneDSInspector/EventInspector.cs +++ /dev/null @@ -1,241 +0,0 @@ -// -// Copyright (c) Microsoft Corporation. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// - -[assembly: Fiddler.RequiredVersion("2.3.0.0")] - -namespace OneDSInspector -{ - using Fiddler; - using System; - using System.Collections.Generic; - using System.Linq; - using System.Text; - using System.Windows.Forms; - using Decoder = CommonSchema.Decoder; - - public class EventInspector : Inspector2, IRequestInspector2 - { - // Inspector2 Overrides/Implementation - - /// - /// Gets the order for this inspector. - /// - /// The inspector order - public override int GetOrder() - { - return 0; - } - - /// - /// Adds our control to the specified tab. - /// - /// The tab page to add to - public override void AddToTab(TabPage page) - { - page.Text = "1DS Events"; - page.Controls.Add(this.PayloadViewerControl); - page.Controls[0].Dock = DockStyle.Fill; - } - - //// IRequestInspector2 Implementation - - /// - /// Gets or sets the request headers (null to disallow editing). - /// - public HTTPRequestHeaders headers - { - get - { - return null; - } - - set - { - headersCopy = value.ToDictionary(key => key.Name, key => key.Value); - - if (!CheckIfPathHasDetails(value)) - { - // Inspect the header for ClientId to see whether it uses Auth or No-Auth. - // This is needed for body decoding. - this.ClientId = GetHeaderValue(value, "Client-Id"); - - // Inspect the content type to determine whether it is compact binary v1 or v2 - this.ContentType = GetHeaderValue(value, "Content-Type"); - - // Inspect the content encoding type to determine whether gunzip is needed - this.ContentEncoding = GetHeaderValue(value, "Content-Encoding"); - } - } - - } - - /// - /// Clear the inspector contents. - /// - public void Clear() - { - this.DecodedJson = string.Empty; - this.PayloadViewerControl.SetText(new List()); - } - - /// - /// Gets a value indicating whether the inspector is dirty (edited); - /// - public bool bDirty - { - get - { - return false; - } - } - - /// - /// Gets or sets whether the inspector is read-only. - /// - public bool bReadOnly - { - get - { - return true; - } - - set - { - } - } - - /// - /// Gets or sets the body. - /// - public byte[] body - { - get - { - return Encoding.UTF8.GetBytes(this.DecodedJson); - } - - set - { - this.SavedBody = value; - UpdatePayloadView(); - } - } - - // Private Implementation - private Dictionary headersCopy; - - /// - /// Gets or sets the text control where the content is visible. - /// - private PayloadViewer PayloadViewerControl { get; set; } - - /// - /// Gets or sets the inspected JSON content. - /// - private string DecodedJson { get; set; } - - /// - /// Gets or sets the "Client-Id" of the inspected content. - /// - private string ClientId { get; set; } - - /// - /// Gets or sets the "Content-Type" of the inspected content. - /// - private string ContentType { get; set; } - - /// - /// Gets or sets the "Content-Encoding" of the inspected content. - /// - private string ContentEncoding { get; set; } - - /// - /// Last body array value we received. - /// This is used if we need to re-parse it due to changed viewing prefrences. - /// - private byte[] SavedBody { get; set; } - - /// - /// Creates a new RecordInspector. - /// - public EventInspector() - { - this.PayloadViewerControl = new PayloadViewer(); - this.DecodedJson = string.Empty; - this.PayloadViewerControl.ViewerConfigurationChanged += (obj, eventtArgs) => { UpdatePayloadView(); }; - } - - /// - /// Checks the path to see if the header details are specified on the path. If the details exist, then - /// it will populate the class properties with the values. - /// - /// The HTTP request headers - /// True, if header details are on the path, false otherwise. - private bool CheckIfPathHasDetails(HTTPRequestHeaders headers) - { - //determine if it is indeed send by the SDK - if (headers.RequestPath.Contains("qsp=true")) - { - //Find where the header details will start - var headerDetailsStart = headers.RequestPath.IndexOf("&"); - if (headerDetailsStart > 0 && headerDetailsStart < headers.RequestPath.Length) - { - String[] headersDetails = headers.RequestPath.Substring(headerDetailsStart + 1).Split('&'); - foreach (var header in headersDetails) - { - String[] headerKeyAndValue = header.Split('='); - if (headerKeyAndValue.Length == 2) - { - if (headerKeyAndValue[0].Equals("client-id", StringComparison.OrdinalIgnoreCase)) - { - this.ClientId = headerKeyAndValue[1]; - } - else if (headerKeyAndValue[0].Equals("content-type", StringComparison.OrdinalIgnoreCase)) - { - this.ContentType = Uri.UnescapeDataString(headerKeyAndValue[1]); - } - } - } - } - return true; - } - return false; - } - - /// - /// Lookup the value of the specified header. - /// - /// The HTTP request headers - /// The name of the header - /// The header value, or an empty string if it is not found - private static string GetHeaderValue(HTTPRequestHeaders headers, string headerName) - { - var header = headers.FirstOrDefault(h => h.Name.Equals(headerName, StringComparison.InvariantCultureIgnoreCase)); - return header != null ? header.Value : string.Empty; - } - - public void UpdatePayloadView() - { - byte[] body = this.SavedBody; - List jsonList; - bool outputCompactJson = this.PayloadViewerControl.IsCompactJsonOutputRequested(); - - try - { - // var items = headers.ToArray(); - Decoder decoder = new Decoder(headersCopy, body); - string json = decoder.ToJson(outputCompactJson, true, (outputCompactJson) ? -1 : 2); - jsonList = json.Split(new string[] { Environment.NewLine }, StringSplitOptions.None).ToList(); - } - catch (Exception exception) - { - // if an exception occurs, display it in the inspector body - jsonList = new List(); - jsonList.Add(exception.Message); - } - this.PayloadViewerControl.SetText(jsonList); - } - } -} \ No newline at end of file diff --git a/tools/fiddler/OneDSInspector/Fiddler.exe.config b/tools/fiddler/OneDSInspector/Fiddler.exe.config deleted file mode 100644 index 8b63649d2..000000000 --- a/tools/fiddler/OneDSInspector/Fiddler.exe.config +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/tools/fiddler/OneDSInspector/OneDSInspector.csproj b/tools/fiddler/OneDSInspector/OneDSInspector.csproj deleted file mode 100644 index 935ff229f..000000000 --- a/tools/fiddler/OneDSInspector/OneDSInspector.csproj +++ /dev/null @@ -1,96 +0,0 @@ - - - - - Debug - AnyCPU - {B1916B1C-9108-4965-8D22-AD840CF2C059} - Library - - OneDSInspector - OneDSInspector - v4.6.1 - 512 - - - - - - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - true - true - - - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - MixedRecommendedRules.ruleset - false - false - - - true - - - - ..\..\packages\Bond.Core.CSharp.9.0.3\lib\net46\Bond.dll - - - ..\..\packages\Bond.Core.CSharp.9.0.3\lib\net46\Bond.Attributes.dll - - - ..\..\packages\Bond.Core.CSharp.9.0.3\lib\net46\Bond.IO.dll - - - ..\..\packages\Bond.Runtime.CSharp.9.0.3\lib\net45\Bond.JSON.dll - - - ..\..\packages\Bond.Core.CSharp.9.0.3\lib\net46\Bond.Reflection.dll - - - $(LOCALAPPDATA)\Programs\Fiddler\Fiddler.exe - - - ..\..\packages\Newtonsoft.Json.11.0.2\lib\net45\Newtonsoft.Json.dll - - - - - - - - - - UserControl - - - - - - - - - - - - - - - - - - copy $(ProjectDir)$(OutDir)OneDSInspector.dll "$(LOCALAPPDATA)\Programs\Fiddler\Inspectors" -copy $(ProjectDir)$(OutDir)OneDSInspector.pdb "$(LOCALAPPDATA)\Programs\Fiddler\Inspectors" -copy $(ProjectDir)ref\*.dll "$(LOCALAPPDATA)\Programs\Fiddler\Inspectors" - - - \ No newline at end of file diff --git a/tools/fiddler/OneDSInspector/PayloadViewer.cs b/tools/fiddler/OneDSInspector/PayloadViewer.cs deleted file mode 100644 index 62704d447..000000000 --- a/tools/fiddler/OneDSInspector/PayloadViewer.cs +++ /dev/null @@ -1,119 +0,0 @@ -// -// Copyright (c) Microsoft Corporation. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -using System; -using System.Collections.Generic; -using System.Windows.Forms; - -namespace OneDSInspector -{ - public partial class PayloadViewer : UserControl - { - /// - /// Required designer variable. - /// - private System.ComponentModel.IContainer components = null; - - private CheckBox m_compactJsonCheckBox; - private RichTextBox m_mainText; - - public event EventHandler ViewerConfigurationChanged; - - /// - /// Clean up any resources being used. - /// - /// true if managed resources should be disposed; otherwise, false. - protected override void Dispose(bool disposing) - { - if (disposing && (components != null)) - { - components.Dispose(); - } - base.Dispose(disposing); - } - - public PayloadViewer() - { - InitializeComponent(); - } - - public void SetText(List textList) - { - // determine the text width and set our width accordingly - // this allows to extend control beyond the default ~3510 characters - - int maxWidth = 0; - - for (int i = 0; i < textList.Count; i++) - { - int curWidth = TextRenderer.MeasureText(textList[i], this.m_mainText.Font).Width; - maxWidth = (maxWidth < curWidth ? curWidth : maxWidth); - } - - // adding marging here is a magic workaround - adding 7 to MarginRight ensures that the text does not wrap - this.m_mainText.RightMargin = maxWidth + this.m_mainText.Margin.Left + this.m_mainText.Margin.Right + 1; - - this.m_mainText.Lines = textList.ToArray(); - } - - public bool IsCompactJsonOutputRequested() - { - return m_compactJsonCheckBox.Checked; - } - - private void PayloadViewer_Layout(object sender, LayoutEventArgs e) - { - m_mainText.Width = Math.Max(0, this.Width - 2 * m_mainText.Left); - m_mainText.Height = Math.Max(0, this.Height - 2 * m_mainText.Top); - } - - private void InitializeComponent() - { - this.m_compactJsonCheckBox = new System.Windows.Forms.CheckBox(); - this.m_mainText = new System.Windows.Forms.RichTextBox(); - this.SuspendLayout(); - // - // m_compactJsonCheckBox - // - this.m_compactJsonCheckBox.AutoSize = true; - this.m_compactJsonCheckBox.Location = new System.Drawing.Point(0, 0); - this.m_compactJsonCheckBox.Name = "m_compactJsonCheckBox"; - this.m_compactJsonCheckBox.Size = new System.Drawing.Size(174, 17); - this.m_compactJsonCheckBox.TabIndex = 0; - this.m_compactJsonCheckBox.Text = "Produce compact JSON output"; - this.m_compactJsonCheckBox.UseVisualStyleBackColor = true; - this.m_compactJsonCheckBox.CheckedChanged += new System.EventHandler(this.m_compactJsonCheckBox_CheckedChanged); - // - // m_mainText - // - this.m_mainText.BackColor = System.Drawing.SystemColors.Window; - this.m_mainText.Font = new System.Drawing.Font("Courier New", 8.25F); - this.m_mainText.Location = new System.Drawing.Point(0, 25); - this.m_mainText.Name = "m_mainText"; - this.m_mainText.ReadOnly = true; - this.m_mainText.Size = new System.Drawing.Size(300, 200); - this.m_mainText.TabIndex = 0; - this.m_mainText.Text = ""; - this.m_mainText.WordWrap = false; - // - // PayloadViewer - // - this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); - this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - this.Controls.Add(this.m_compactJsonCheckBox); - this.Controls.Add(this.m_mainText); - this.Name = "PayloadViewer"; - this.Size = new System.Drawing.Size(303, 203); - this.Layout += new System.Windows.Forms.LayoutEventHandler(this.PayloadViewer_Layout); - this.ResumeLayout(false); - this.PerformLayout(); - - } - - private void m_compactJsonCheckBox_CheckedChanged(object sender, EventArgs e) - { - this.ViewerConfigurationChanged?.Invoke(this, e); - } - } -} diff --git a/tools/fiddler/OneDSInspector/PayloadViewer.resx b/tools/fiddler/OneDSInspector/PayloadViewer.resx deleted file mode 100644 index 1af7de150..000000000 --- a/tools/fiddler/OneDSInspector/PayloadViewer.resx +++ /dev/null @@ -1,120 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - \ No newline at end of file diff --git a/tools/fiddler/OneDSInspector/README.md b/tools/fiddler/OneDSInspector/README.md index c5280889b..c1ce252fd 100644 --- a/tools/fiddler/OneDSInspector/README.md +++ b/tools/fiddler/OneDSInspector/README.md @@ -1,52 +1,3 @@ -# 1DS Bond Inspector for Fiddler +# OneDSInspector -This is 1DS Common Schema / Bond decoder for Fiddler. - -Please build and deploy to *%LOCALAPPDATA%\Programs\Fiddler\Inspectors* - -# How to use - -- download and install Fiddler -- make sure SSL decoding is enabled (Collector uses TLS 1.2+) -- ingest some 1DS test traffic from a test app, e.g. EventSender example -- navigate to the list of HTTP requests -- filter by your collector end-point or your test app name -- click on collector HTTP request of interest -- click on *Inspectors* tab -> *1DS Events* tab -- enjoy an event decoded from Bond to human-readable JSON -- copy-paste JSON to pretty-print, OR -- use the built-in indented format - -# Fiddler.exe.config - -You may occasionally encounter a `Newtonsoft.JSON` assembly loading issue with a specific version of Newtonsoft/Fiddler. -If and when that happens - 1DS inspector should show an exception in the decoded output window. -This could happen when a more recent Fiddler is installed with a version of Newtonsoft that differs from the one used by 1DS Bond Inspector plugin. - -Exit Fiddler first. - -Then use the Binding Redirect feature as follows in *Fiddler.exe.config* to redirect to the currently deployed version: - -``` - - - - - - - - - - - - - - - -``` -In example above - we redirected the Newtonsoft.Json assembly version from 0-12 to 11 used by Fiddler. -Please make sure you redirect to version employed by your Fiddler installation. -Place this file above (Fiddler.exe.config) to %LOCALAPPDATA%\Programs\Fiddler\ next to Fiddler.exe -Then restart Fiddler.exe . This should solve the Newtonsoft loading issue. - -Happy Fiddling! +This tool is no longer maintained, but you can get it from the [history](https://github.com/microsoft/cpp_client_telemetry/tree/2c08cbe3d0541888334046cc224bc6a1e1b72aa7/tools/fiddler/OneDSInspector) for local testing. \ No newline at end of file diff --git a/tools/fiddler/OneDSInspector/app.config b/tools/fiddler/OneDSInspector/app.config deleted file mode 100644 index 51e4cb971..000000000 --- a/tools/fiddler/OneDSInspector/app.config +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/tools/fiddler/OneDSInspector/packages.config b/tools/fiddler/OneDSInspector/packages.config deleted file mode 100644 index 4c1252c00..000000000 --- a/tools/fiddler/OneDSInspector/packages.config +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/tools/fiddler/OneDSInspector/ref/dummy.dll b/tools/fiddler/OneDSInspector/ref/dummy.dll deleted file mode 100644 index e69de29bb..000000000 diff --git a/tools/server/.gitignore b/tools/server/.gitignore deleted file mode 100644 index 116e29ae7..000000000 --- a/tools/server/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -# Don't self ignore -!.gitignore -!.tfignore - -# Default project build output dirs -bin/ -obj/ -csx/ - -# jsoneditor -jsoneditor/ - -# Miscellaneous files to ignore -*.suo -*.user -*.sdf -*.opensdf -*.backup.* -*.DotSettings -*\rcf - -.vscode/ -server.code-workspace -/jsoneditor/ diff --git a/tools/server/Program.cs b/tools/server/Program.cs deleted file mode 100644 index 3f183a3c0..000000000 --- a/tools/server/Program.cs +++ /dev/null @@ -1,71 +0,0 @@ -// -// Copyright (c) Microsoft Corporation. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -// System -// AspNetCore -using Microsoft.AspNetCore; -using Microsoft.AspNetCore.Hosting; -// Configuration and Logging -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using System.IO; - -namespace CommonSchema.Server -{ - /// - /// Reference implementation of Common Schema protocol server. - /// - public class Program - { - - /// - /// Program main entry point - /// - /// - public static void Main(string[] args) - { - CreateWebHostBuilder(args) - .UseKestrel(options => - { - options.AllowSynchronousIO = true; - }) - .UseContentRoot(Directory.GetCurrentDirectory()) - .UseWebRoot(Path.Combine(Directory.GetCurrentDirectory(), "wwwroot")) - .ConfigureAppConfiguration((hostingContext, config) => - { - IWebHostEnvironment env = hostingContext.HostingEnvironment; - IConfigurationBuilder configurationBuilder = config - .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) - .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true, reloadOnChange: true) - .AddEnvironmentVariables(); - var fileName = configurationBuilder.Build().GetSection("FileNameToStoreTelemetryData")?.Value; - // clean the file before every launch of the server - if (!string.IsNullOrEmpty(fileName) && File.Exists(fileName)) File.Delete(fileName); - - }) - .ConfigureLogging(configureLogging: (hostingContext, logging) => - { - // Requires `using Microsoft.Extensions.Logging;` - logging.AddConfiguration(hostingContext.Configuration.GetSection("Logging")); - logging.AddConsole(); - logging.AddDebug(); - logging.AddEventSourceLogger(); - }) - .Build() - .Run(); - } - - /// - /// Creates instance of WebHost using Startup - /// - /// - /// IWebHostBuilder web host instance - public static IWebHostBuilder CreateWebHostBuilder(string[] args) - { - return WebHost - .CreateDefaultBuilder(args) - .UseStartup(); - } - } -} diff --git a/tools/server/README.md b/tools/server/README.md deleted file mode 100644 index 2994d129e..000000000 --- a/tools/server/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# 1DS Test Server - -Test server requires ASP.NET Core 3.x. Uses [Kestrel](https://docs.microsoft.com/en-us/aspnet/core/fundamentals/servers/kestrel?view=aspnetcore-3.1) as default server. - -This test server is provided as a reference, for test purposes only, not supported, and not intended to be used by production services. - -## Usage - -`run.cmd` - on Windows -`run.sh` - on Linux and Mac - -Or launch it directly from Visual Studio solution (tools\tools.sln). - -## Directing client SDK to test server - -```cpp - auto& config = LogManager::GetLogConfiguration(); - config[CFG_STR_COLLECTOR_URL] = "http://localhost:5000/OneCollector/"; -``` - -Emit some events. Observe JSON-decoded events in test server console. diff --git a/tools/server/Startup.cs b/tools/server/Startup.cs deleted file mode 100644 index 883771d45..000000000 --- a/tools/server/Startup.cs +++ /dev/null @@ -1,178 +0,0 @@ -// -// Copyright (c) Microsoft Corporation. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Primitives; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Configuration; - -namespace CommonSchema -{ - namespace Server - { - public class Startup - { - private int seq = 0; - private static object locker = new Object(); - - public Startup(IConfiguration configuration) - { - Configuration = configuration; - } - - public IConfiguration Configuration { get; } - - // This method gets called by the runtime. Use this method to add services to the container. - // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 - public void ConfigureServices(/* IServiceCollection services */) - { - // TODO: add services configuration - } - - // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. - public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerFactory loggerFactory) - { - if (env.IsDevelopment()) - { - app.UseDeveloperExceptionPage(); - } - - app.UseStaticFiles(); // For the wwwroot folder - app.Run(async (context) => - { - // Dump request information with headers - seq++; - ILogger requestLogger = loggerFactory.CreateLogger("REQ-" + seq); - ILogger decoderLogger = loggerFactory.CreateLogger("DEC-" + seq); - - string headersStr = ""; - foreach (var entry in context.Request.Headers) - { - headersStr += entry.Key + ": " + entry.Value.ToString() + "\n"; - } - requestLogger.LogInformation(headersStr); - - try - { - string path = context.Request.Path.Value; - if (path.StartsWith("/OneCollectorWriteToFile")) - { - string result = DecodeTelemetryRequest(context, decoderLogger); - var fileName = Configuration.GetSection("FileNameToStoreTelemetryData")?.Value; - if (!string.IsNullOrEmpty(fileName)) - { - lock (locker) - { - if (File.Exists(fileName)) - { - var formattedResult = result.Replace("[", "").Replace("]", ""); - var currentContent = File.ReadAllText(fileName); - var updatedContent = string.Concat(currentContent.Replace("]", ","), formattedResult, "]"); - File.WriteAllText(fileName, updatedContent); - } - else - { - File.AppendAllText(fileName, result); - } - } - } - else - { - requestLogger.LogError("Parameter FileNameToStoreTelemetryData from appsettings.json, where data should be stored is not specified. As a result telemetry data won't be saved to file, but will be visible in the console"); - } - - // Echo the body converted to JSON array - context.Response.StatusCode = 200; - requestLogger.LogInformation(result); - await context.Response.WriteAsync(result); - } - else - if (path.StartsWith("/OneCollector/")) - { - string result = DecodeTelemetryRequest(context, decoderLogger); - - // Echo the body converted to JSON array - context.Response.StatusCode = 200; - requestLogger.LogInformation(result); - await context.Response.WriteAsync(result); - } - else - /* Azure Monitor / Application Insights -compatible server */ - if (path.StartsWith("/v2/track")) - { - int length = Int32.Parse(context.Request.Headers["Content-Length"]); - BinaryReader reader = new BinaryReader(context.Request.Body); - - // Read body fully before decoding it - byte[] buffer = reader.ReadBytes(length); - - if (context.Request.Headers["Content-Encoding"] == "gzip") - { - buffer = Decoder.Gunzip(buffer); - } - else - if (context.Request.Headers["Content-Encoding"] == "deflate") - { - buffer = Decoder.Inflate(buffer); - } - - string result = System.Text.Encoding.UTF8.GetString(buffer); - - // Echo the body converted to JSON array - context.Response.StatusCode = 200; - requestLogger.LogInformation(result); - await context.Response.WriteAsync(result); - } - else - if (path.StartsWith("/admin/stop")) - { - // Stop web-server - requestLogger.LogInformation("Stopping web-server..."); - context.Response.StatusCode = 200; - await context.Response.WriteAsync("Server stopped."); - Environment.Exit(0); - } - } - catch (Exception ex) - { - // Handle all error conditions here - string result = "400 Bad Request"; - context.Response.StatusCode = 400; - requestLogger.LogError("Exception: {ex}", ex); - await context.Response.WriteAsync(result); - } - }); - - } - - private static string DecodeTelemetryRequest(HttpContext context, ILogger decoderLogger) - { - int length = Int32.Parse(context.Request.Headers["Content-Length"]); - BinaryReader reader = new BinaryReader(context.Request.Body); - - // Read body fully before decoding it - byte[] buffer = reader.ReadBytes(length); - - Dictionary headers = new Dictionary(); - foreach (KeyValuePair entry in context.Request.Headers) - { - // Our decoder only needs to know the 1st header value, do not need a multimap - headers[entry.Key] = entry.Value.ElementAt(0); - }; - Decoder decoder = new Decoder(headers, buffer); - // Supply the logger - decoder.Logger = decoderLogger; - string result = decoder.ToJson(false, true, 2); - return result; - } - } - } -} diff --git a/tools/server/bond/CSProtocol.bond b/tools/server/bond/CSProtocol.bond deleted file mode 100644 index fd8b6283a..000000000 --- a/tools/server/bond/CSProtocol.bond +++ /dev/null @@ -1,299 +0,0 @@ -// Generated from https://skype.visualstudio.com/aria/_git/infrastructure_data_pipe?path=%2FServices%2FModels%2FCommonSchema%2FBond%2FCommonSchemaEvent.bond -// by applying the following steps: -// 1) Take CommonSchemaEvent.bond and rename it to CommonSchema.bond. -// 2) Remove comments, Bond metadata, BOM-marker, normalize tabs and linebreaks. -// 3) Comment out recursive Ingest.processedIngest field. -// 4) Change namespace to CommonSchema. -// 5) Change CommonSchemaEvent struct name to CsEvent. -// 6) Replace 'nullable' with 'vector'. -// 7) Remove default 'nothing' for strings to avoid maybe<>. -// 8) Remove bonded<>. - -namespace CommonSchema - -struct Ingest -{ - 1: required int64 time; - 2: required string clientIp; - 3: optional int64 auth; - 4: optional int64 quality; - 5: optional int64 uploadTime; - 6: optional string userAgent; - 7: optional string client; -// 8: optional Ingest processedIngest; -} - -struct User -{ - 1: optional string id; - 2: optional string localId; - 3: optional string authId; - 4: optional string locale; -} - -struct Loc -{ - 1: optional string id; - 2: optional string country; - 3: optional string timezone; -} - -struct Device -{ - 1: optional string id; - 2: optional string localId; - 3: optional string authId; - 4: optional string authSecId; - 5: optional string deviceClass; - 6: optional string orgId; - 7: optional string orgAuthId; - 8: optional string make; - 9: optional string model; -} - -struct Os -{ - 1: optional string locale; - 2: optional string expId; - 3: optional int32 bootId; - 4: optional string name; - 5: optional string ver; -} - -struct App -{ - 1: optional string expId; - 2: optional string userId; - 3: optional string env; - 4: optional int32 asId; - 5: optional string id; - 6: optional string ver; - 7: optional string locale; - 8: optional string name; -} - -struct Utc -{ - 1: optional string stId; - 2: optional string aId; - 3: optional string raId; - 4: optional string op; - 5: optional int64 cat; - 6: optional int64 flags; - 7: optional string sqmId; - 9: optional string mon; - 10: optional int32 cpId; - 11: optional string bSeq; - 12: optional string epoch; - 13: optional int64 seq; - // cs4.0 fields - 14: optional double popSample; - 15: optional int64 eventFlags; -} - -struct M365a -{ - 1: optional string enrolledTenantId; -} - -struct Xbl -{ - 5: optional map claims; - 10: optional string nbf; - 20: optional string exp; - 30: optional string sbx; - 40: optional string dty; - 50: optional string did; - 60: optional string xid; - 70: optional uint64 uts; - 80: optional string pid; - 90: optional string dvr; - 100: optional uint32 tid; - 110: optional string tvr; - 120: optional string sty; - 130: optional string sid; - 140: optional int64 eid; - 150: string ip; -} - -struct Javascript -{ - 10: optional string libVer; - 15: optional string osName; - 20: optional string browser; - 21: optional string browserVersion; - 25: optional string platform; - 30: optional string make; - 35: optional string model; - 40: optional string screenSize; - 50: optional string mc1Id; - 60: optional uint64 mc1Lu; - 70: optional bool isMc1New; - 80: optional string ms0; - 90: optional string anid; - 100: optional string a; - 110: optional string msResearch; - 120: optional string csrvc; - 130: optional string rtCell; - 140: optional string rtEndAction; - 150: optional string rtPermId; - 160: optional string r; - 170: optional string wtFpc; - 180: optional string omniId; - 190: optional string gsfxSession; - 200: optional string domain; - 999: optional string dnt; -} - -struct Protocol -{ - 1: optional int32 metadataCrc; - 2: optional vector> ticketKeys; - 3: optional string devMake; - 4: optional string devModel; -} - -struct Receipts -{ - 1: optional int64 originalTime; - 2: optional int64 uploadTime; -} - -struct Net -{ - 1: optional string provider; - 2: optional string cost; - 3: optional string type; -} - -struct Sdk -{ - 1: optional string libVer; - 2: optional string epoch; - 3: optional int64 seq; - 4: optional string installId; -} - -struct Cloud -{ - 1: optional string fullEnvName; - 2: optional string location; - 3: optional string environment; - 4: optional string deploymentUnit; - 5: optional string name; - 6: optional string roleInstance; - 7: optional string role; -} - - -enum ValueKind -{ - ValueInt64 = 0, - ValueUInt64 = 1, - ValueInt32 = 2, - ValueUInt32 = 3, - ValueDouble = 4, - ValueString = 5, - ValueBool = 6, - ValueDateTime = 7, - ValueGuid = 8, - ValueArrayInt64 = 9, - ValueArrayUInt64 = 10, - ValueArrayInt32 = 11, - ValueArrayUInt32 = 12, - ValueArrayDouble = 13, - ValueArrayString = 14, - ValueArrayBool = 15, - ValueArrayDateTime = 16, - ValueArrayGuid = 17 -} - -enum PIIKind -{ - NotSet = 0, - DistinguishedName = 1, - GenericData = 2, - IPV4Address = 3, - IPv6Address = 4, - MailSubject = 5, - PhoneNumber = 6, - QueryString = 7, - SipAddress = 8, - SmtpAddress = 9, - Identity = 10, - Uri = 11, - Fqdn = 12, - IPV4AddressLegacy = 13 -} - -struct PII -{ - 1: optional PIIKind Kind = NotSet; -} - -enum CustomerContentKind -{ - NotSet = 0, - GenericContent = 1 -} - -struct CustomerContent -{ - 1: optional CustomerContentKind Kind = NotSet; -} - -struct Attributes -{ - 1: optional vector pii; - 2: optional vector customerContent; -} - -struct Value -{ - 1: optional ValueKind type = ValueString; - 2: optional vector attributes; - 3: optional string stringValue; - 4: optional int64 longValue; // all integer types, data time, bool is represented here - 5: optional double doubleValue; - 6: optional vector> guidValue; - 10: optional vector> stringArray; - 11: optional vector> longArray; // all vectors of integer types, data time, bool are represented here - 12: optional vector> doubleArray; - 13: optional vector>> guidArray; -} - -struct Data -{ - 1: optional map properties; -} - -struct CsEvent -{ - 1: required string ver; - 2: required string name; - 3: required int64 time; - 4: optional double popSample = 100.0; - 5: optional string iKey; - 6: optional int64 flags; - 7: optional string cV; - 20: optional vector extIngest; - 21: optional vector extProtocol; - 22: optional vector extUser; - 23: optional vector extDevice; - 24: optional vector extOs; - 25: optional vector extApp; - 26: optional vector extUtc; - 27: optional vector extXbl; - 28: optional vector extJavascript; - 29: optional vector extReceipts; - 31: optional vector extNet; - 32: optional vector extSdk; - 33: optional vector extLoc; - 34: optional vector extCloud; - 37: optional vector extM365a; - 41: optional vector ext; - 51: optional map tags; - 60: optional string baseType; - 61: optional vector baseData; - 70: vector data; -} \ No newline at end of file diff --git a/tools/server/bond/README.md b/tools/server/bond/README.md deleted file mode 100644 index 16b9d1315..000000000 --- a/tools/server/bond/README.md +++ /dev/null @@ -1,4 +0,0 @@ -This directory shows how to generate C# code from .bond schema files using gbc.exe -The code can be subsequently reused for building the tool on other OS. -TODO: find the tool to generate the source on non-Windows platform. - diff --git a/tools/server/bond/bond.bond b/tools/server/bond/bond.bond deleted file mode 100644 index e69de29bb..000000000 diff --git a/tools/server/bond/bond_const.bond b/tools/server/bond/bond_const.bond deleted file mode 100644 index 4d7c54eef..000000000 --- a/tools/server/bond/bond_const.bond +++ /dev/null @@ -1,59 +0,0 @@ -namespace bond - -// Enumerator of Bond meta-schema types -enum BondDataType -{ - BT_STOP = 0, - BT_STOP_BASE = 1, - BT_BOOL = 2, - BT_UINT8 = 3, - BT_UINT16 = 4, - BT_UINT32 = 5, - BT_UINT64 = 6, - BT_FLOAT = 7, - BT_DOUBLE = 8, - BT_STRING = 9, - BT_STRUCT = 10, - BT_LIST = 11, - BT_SET = 12, - BT_MAP = 13, - BT_INT8 = 14, - BT_INT16 = 15, - BT_INT32 = 16, - BT_INT64 = 17, - BT_WSTRING = 18, - BT_UNAVAILABLE= 127 -} - -// Enumerator to distinguish the different subtypes treated as BT_LIST -enum ListSubType -{ - // Used when the type is a list/vector. - // Also used when the subtype cannot be determined. - NO_SUBTYPE = 0, - - // Used when the type is nullable. - NULLABLE_SUBTYPE = 1, - - // Used when the type is a blob. - BLOB_SUBTYPE = 2 -} - -// Magic numbers of predefined protocols -enum ProtocolType -{ - // Actual protocol type is marshaled with payload - MARSHALED_PROTOCOL = 0, - - // Fast binary protocol - FAST_PROTOCOL = 0x464d, - - // Compact binary protocol - COMPACT_PROTOCOL = 0x4243, - - // Simple JSON protocol - SIMPLE_JSON_PROTOCOL = 0x4a53, - - // Simple binary protocol - SIMPLE_PROTOCOL = 0x5053, -} diff --git a/tools/server/bond/compile.cmd b/tools/server/bond/compile.cmd deleted file mode 100644 index cf5a057f0..000000000 --- a/tools/server/bond/compile.cmd +++ /dev/null @@ -1,5 +0,0 @@ -where /r .. gbc.exe > %TEMP%\gbc.path -set /P GBC=<%TEMP%\gbc.path -%GBC% c# CSProtocol.bond -move *.cs .. - diff --git a/tools/server/run.cmd b/tools/server/run.cmd deleted file mode 100644 index f1d297ddd..000000000 --- a/tools/server/run.cmd +++ /dev/null @@ -1,29 +0,0 @@ -@ECHO OFF -set framework="netcoreapp3.1" - -:loop -IF NOT "%1"=="" ( - IF "%1"=="-f" ( - SET framework=%2 - SHIFT - ) - IF "%1"=="--framework" ( - SET framework=%2 - SHIFT - ) - IF "%1"=="-h" GOTO :help - IF "%1"=="--help" GOTO :help - SHIFT - GOTO :loop -) -echo "1DS test telemetry server will be run with %framework%" -dotnet run --framework %framework% -goto:eof - -:help - echo "Run 1DS Test Telemetry Server with specified runtime versions (.net core 3 or .net6)." - echo "Syntax: run.sh [-f,--framework|-h, --help]" - echo "options:" - echo "-f | --framework Run server with specified version of runtime. Supported values: netcoreapp3.1 or net6.0. Default value is netcoreapp3.1, if option is not specified" - echo "-h | --help Help." - goto:eof \ No newline at end of file diff --git a/tools/server/run.sh b/tools/server/run.sh deleted file mode 100755 index d1cc32ed3..000000000 --- a/tools/server/run.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh - -############################################################ -# Help # -############################################################ -Help() -{ - # Display Help - echo "Run 1DS Test Telemetry Server with specified runtime versions (.net core 3 or .net6)." - echo - echo "Syntax: run.sh [-f,--framework|-h, --help]" - echo "options:" - echo "-f | --framework Run server with specified version of runtime. Supported values: netcoreapp3.1 or net6.0. Default value is netcoreapp3.1, if option is not specified" - echo "-h | --help Help." - echo - - exit 0 -} - -framework="netcoreapp3.1" -while test $# -gt 0; -do - case "$1" in - -h|--help) Help;; - -f|--framework) - shift - if test $# -gt 0; then - framework=$1 - else - echo "--framework option value is not specified. Server will be run with $framework" - fi - shift - ;; - *) break;; - esac -done -echo "1DS test telemetry server will be run with $framework" -dotnet run -f $framework - diff --git a/tools/server/server.csproj b/tools/server/server.csproj deleted file mode 100644 index 2885d9e03..000000000 --- a/tools/server/server.csproj +++ /dev/null @@ -1,27 +0,0 @@ - - - - netcoreapp3.1;net6.0 - true - true - true - - - - TRACE;NETCOREAPP - - - - - - - - - - - - - - - - diff --git a/tools/tools.sln b/tools/tools.sln deleted file mode 100644 index 2f4dd1583..000000000 --- a/tools/tools.sln +++ /dev/null @@ -1,41 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.30204.135 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OneDSInspector", "fiddler\OneDSInspector\OneDSInspector.csproj", "{B1916B1C-9108-4965-8D22-AD840CF2C059}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "server", "server\server.csproj", "{7627E313-4B14-43E6-8ED7-0880C3657084}" -EndProject -Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "decoder", "decoder\decoder.shproj", "{6A206BE3-9975-4952-8E1D-35BC2F47431C}" -EndProject -Global - GlobalSection(SharedMSBuildProjectFiles) = preSolution - decoder\decoder.projitems*{6a206be3-9975-4952-8e1d-35bc2f47431c}*SharedItemsImports = 13 - decoder\decoder.projitems*{7627e313-4b14-43e6-8ed7-0880c3657084}*SharedItemsImports = 5 - decoder\decoder.projitems*{b1916b1c-9108-4965-8d22-ad840cf2c059}*SharedItemsImports = 4 - EndGlobalSection - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {B1916B1C-9108-4965-8D22-AD840CF2C059}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B1916B1C-9108-4965-8D22-AD840CF2C059}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B1916B1C-9108-4965-8D22-AD840CF2C059}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B1916B1C-9108-4965-8D22-AD840CF2C059}.Release|Any CPU.Build.0 = Release|Any CPU - {7627E313-4B14-43E6-8ED7-0880C3657084}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {7627E313-4B14-43E6-8ED7-0880C3657084}.Debug|Any CPU.Build.0 = Debug|Any CPU - {7627E313-4B14-43E6-8ED7-0880C3657084}.Release|Any CPU.ActiveCfg = Release|Any CPU - {7627E313-4B14-43E6-8ED7-0880C3657084}.Release|Any CPU.Build.0 = Release|Any CPU - {6A206BE3-9975-4952-8E1D-35BC2F47431C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {6A206BE3-9975-4952-8E1D-35BC2F47431C}.Release|Any CPU.ActiveCfg = Release|Any CPU - {6A206BE3-9975-4952-8E1D-35BC2F47431C}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {370C5DF1-DBF0-4505-837E-BDA1ED1D994F} - EndGlobalSection -EndGlobal From 048c9050bf601616eb1675960d956f47da9e6438 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Mon, 13 May 2024 14:06:38 -0700 Subject: [PATCH 4/9] Upgrade CodeQL from v2 to v3 (#1276) --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 65c598dbc..434fbd1c3 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,7 +39,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -50,7 +50,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -64,4 +64,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 From a956eb9cfc7ec2290478207c548652322f29c798 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Mon, 13 May 2024 17:59:08 -0700 Subject: [PATCH 5/9] Add Java to CodeQL scan list (#1275) --- .github/workflows/codeql-analysis.yml | 80 ++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 434fbd1c3..4d8b3ceb6 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -17,7 +17,7 @@ on: jobs: analyze: name: Analyze - runs-on: ubuntu-latest + runs-on: windows-2019 permissions: contents: read actions: read @@ -49,8 +49,17 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v3 + # - name: Autobuild + # uses: github/codeql-action/autobuild@v3 + - name: Build + env: + SKIP_ARM_BUILD: 1 + SKIP_ARM64_BUILD: 1 + PlatformToolset: v142 + VSTOOLS_VERSION: vs2019 + shell: cmd + if: matrix.language == 'cpp' + run: build-all.bat # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -63,5 +72,70 @@ jobs: # make bootstrap # make release + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + + analyze-java: + name: Analyze Java + runs-on: windows-latest + permissions: + contents: read + actions: read + security-events: write + + strategy: + fail-fast: false + + steps: + - name: Checkout + uses: actions/checkout@v2 + continue-on-error: true + + - name: Update submodules + run: | + git submodule sync + git config --global submodule.lib/modules.update none + git -c protocol.version=2 submodule update --init --force --depth=1 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: java + + - name: Setup Java + uses: actions/setup-java@v3 + with: + distribution: 'adopt' + java-version: '11' + - name: Remove default github maven configuration + run: rm $Env:USERPROFILE\.m2\settings.xml + - name: Setup Android SDK + uses: android-actions/setup-android@v2 + - name: Install NDK + run: | + java -version + gci env:* | sort-object name + new-item "C:\Users\runneradmin\.android\repositories.cfg" -ItemType "file" + echo yes | .\sdkmanager.bat "ndk-bundle" "cmake;3.10.2.4988404" "ndk;21.4.7075529" --sdk_root=$Env:ANDROID_SDK_ROOT + working-directory: ${{ env.ANDROID_SDK_ROOT }}\cmdline-tools\7.0\bin + - name: Chocolatey + run: | + choco install --no-progress -y ninja + - name: List CMake + run: | + pwd + echo "==================" + gci -r -i "CMake*" -Name + echo "==================" + gci -r -i "gtest-all*" -Name + echo "==================" + gci third_party/ -Name + echo "==================" + gci third_party/googletest -Name + - name: Gradle Build + run: | + .\gradlew.bat maesdk:assemble app:assemble + working-directory: lib\android_build + - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 From c480743a8603dd34bb8b054877ced15bc48870c1 Mon Sep 17 00:00:00 2001 From: shae-brown <77749213+shae-brown@users.noreply.github.com> Date: Tue, 14 May 2024 10:20:12 -0700 Subject: [PATCH 6/9] Add Json test file for modules (#1271) --- tests/functests/CMakeLists.txt | 3 +++ tests/functests/FuncTests.vcxproj | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/functests/CMakeLists.txt b/tests/functests/CMakeLists.txt index ba0e524e7..99b2c17e3 100644 --- a/tests/functests/CMakeLists.txt +++ b/tests/functests/CMakeLists.txt @@ -34,6 +34,9 @@ if (EXISTS ${CMAKE_SOURCE_DIR}/lib/modules/exp/tests) ${CMAKE_SOURCE_DIR}/lib/modules/exp/tests/functests/ECSClientRealworldFuncTests.cpp ${CMAKE_SOURCE_DIR}/lib/modules/exp/tests/functests/ECSConfigCacheFuncTests.cpp ) + if (EXISTS ${CMAKE_SOURCE_DIR}/lib/modules/exp/tests/functests/test.json) + file(COPY_FILE ${CMAKE_SOURCE_DIR}/lib/modules/exp/tests/functests/test.json ${CMAKE_BINARY_DIR}/test.json) + endif() endif() source_group(" " REGULAR_EXPRESSION "") diff --git a/tests/functests/FuncTests.vcxproj b/tests/functests/FuncTests.vcxproj index 79d3f97c0..d2578806f 100644 --- a/tests/functests/FuncTests.vcxproj +++ b/tests/functests/FuncTests.vcxproj @@ -461,4 +461,7 @@ - + + + + \ No newline at end of file From feff7ef0986a59627906cf7aa009d2f82d0f481e Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Thu, 23 May 2024 08:56:01 -0700 Subject: [PATCH 7/9] Include CSharp sample in build script and add it to CodeQL list (#1278) --- .github/workflows/codeql-analysis.yml | 6 +++--- build-all.bat | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4d8b3ceb6..3861e74f7 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -28,7 +28,7 @@ jobs: matrix: # Override automatic language detection by changing the below list # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] - language: ['cpp', 'javascript', 'python'] + language: ['cpp', 'csharp', 'javascript', 'python'] # Learn more... # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection @@ -58,7 +58,7 @@ jobs: PlatformToolset: v142 VSTOOLS_VERSION: vs2019 shell: cmd - if: matrix.language == 'cpp' + if: matrix.language == 'cpp' || matrix.language == 'csharp' run: build-all.bat # ℹ️ Command-line programs to run using the OS shell. @@ -73,7 +73,7 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 analyze-java: name: Analyze Java diff --git a/build-all.bat b/build-all.bat index ab47fd8b5..6ab0d4ab0 100644 --- a/build-all.bat +++ b/build-all.bat @@ -27,8 +27,8 @@ echo %CUSTOM_PROPS% if NOT DEFINED SKIP_MD_BUILD ( REM DLL and static /MD build REM Release - call tools\RunMsBuild.bat Win32 Release "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,net40:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild" %CUSTOM_PROPS% - call tools\RunMsBuild.bat x64 Release "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,net40:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild" %CUSTOM_PROPS% + call tools\RunMsBuild.bat Win32 Release "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,net40:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild,Samples\cs\SampleCsNet40:Rebuild" %CUSTOM_PROPS% + call tools\RunMsBuild.bat x64 Release "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,net40:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild,Samples\cs\SampleCsNet40:Rebuild" %CUSTOM_PROPS% REM Debug if NOT DEFINED SKIP_DEBUG_BUILD ( call tools\RunMsBuild.bat Win32 Debug "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,net40:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild" %CUSTOM_PROPS% From e93f260c318b95fbbf71a12c98f682b9463ae443 Mon Sep 17 00:00:00 2001 From: Lalit Kumar Bhasin Date: Wed, 29 May 2024 11:26:03 -0700 Subject: [PATCH 8/9] MacOS CI failure fix (#1280) --- build.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index 41628b5ba..f3c7909b0 100755 --- a/build.sh +++ b/build.sh @@ -99,13 +99,15 @@ echo "macosx deployment target="$MACOSX_DEPLOYMENT_TARGET # Install build tools and recent sqlite3 FILE=.buildtools OS_NAME=`uname -a` + if [ ! -f $FILE ]; then case "$OS_NAME" in - *Darwin*) tools/setup-buildtools-apple.sh $MAC_ARCH ;; - *Linux*) [[ -z "$NOROOT" ]] && sudo tools/setup-buildtools.sh || echo "No root: skipping build tools installation." ;; - *) echo "WARNING: unsupported OS $OS_NAME , skipping build tools installation.." + *Darwin*) CMD="tools/setup-buildtools-apple.sh $MAC_ARCH" ;; + *Linux*) CMD="tools/setup-buildtools.sh" ;; + *) CMD=""; echo "WARNING: unsupported OS $OS_NAME, skipping build tools installation.." ;; esac - # Assume that the build tools have been successfully installed + + [[ -n "$CMD" ]] && { [[ -z "$NOROOT" ]] && sudo $CMD || echo "No root: skipping build tools installation."; } echo > $FILE fi From 06c0679cc8bc9511b1ef6edfe564972b69e7e9a1 Mon Sep 17 00:00:00 2001 From: Tomasz Kukielka <113089389+tkukielk@users.noreply.github.com> Date: Wed, 29 May 2024 14:03:49 -0700 Subject: [PATCH 9/9] Fix build scripts and add visionOS support (#1279) * Fix build scripts and add visionOS support Multiple fixes: - building on Apple Silicon Macs - remove the assumptions that the build machine is always x86_64 - fix badly broken build.sh for Mac, especially with getopts for options & params - extend build-ios.sh to allow building for iOS-derived platforms like visionOS The expected iOS build invocation is: build-ios.sh [clean] [release|debug] ${ARCH} ${PLATFORM} where ARCH = arm64|arm64e|x86_64 PLATFORM = iphoneos|iphonesimulator|xros|xrsimulator * Fix build for older Xcodes The verification pipeline uses older Xcode -Wno-unused-but-set-variable is unknown to older compilers, add -Wno-unknown-warning-option * Set the default arch to the build machine's one if not specified --------- Co-authored-by: Tom Tan --- CMakeLists.txt | 22 +++++++---- build-ios.sh | 98 +++++++++++++++++++++++++++++---------------- build.sh | 105 ++++++++++++++++++++++++++++++++----------------- 3 files changed, 146 insertions(+), 79 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b76e4cee1..394ef48e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,22 +46,21 @@ if(APPLE) endif() endif() - if((${IOS_PLAT} STREQUAL "iphoneos") OR (${IOS_PLAT} STREQUAL "iphonesimulator")) + if((${IOS_PLAT} STREQUAL "iphoneos") OR (${IOS_PLAT} STREQUAL "iphonesimulator") OR (${IOS_PLAT} STREQUAL "xros") OR (${IOS_PLAT} STREQUAL "xrsimulator")) set(IOS_PLATFORM "${IOS_PLAT}") else() message(FATAL_ERROR "Unrecognized iOS platform '${IOS_PLAT}'") endif() if(${IOS_ARCH} STREQUAL "x86_64") - set(IOS_PLATFORM "iphonesimulator") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch x86_64") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch x86_64") set(CMAKE_SYSTEM_PROCESSOR x86_64) elseif(${IOS_ARCH} STREQUAL "arm64") - set(IOS_PLATFORM ${IOS_PLAT}) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch arm64") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64") set(CMAKE_SYSTEM_PROCESSOR arm64) elseif(${IOS_ARCH} STREQUAL "arm64e") - set(IOS_PLATFORM ${IOS_PLAT}) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch arm64e") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64e") set(CMAKE_SYSTEM_PROCESSOR arm64e) @@ -78,11 +77,18 @@ if(APPLE) message("-- PLATFORM: ${IOS_PLATFORM}") else() if(${MAC_ARCH} STREQUAL "x86_64") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch x86_64") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch x86_64") set(CMAKE_SYSTEM_PROCESSOR x86_64) + set(TARGET_ARCH ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_OSX_ARCHITECTURES ${MAC_ARCH}) + set(APPLE True) elseif(${MAC_ARCH} STREQUAL "arm64") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch arm64") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64") set(CMAKE_SYSTEM_PROCESSOR arm64) + set(TARGET_ARCH ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_OSX_ARCHITECTURES ${MAC_ARCH}) set(APPLE True) else() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch x86_64 -arch arm64") @@ -107,7 +113,7 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") set(WARN_FLAGS "/W4 /WX") else() # No -pedantic -Wno-extra-semi -Wno-gnu-zero-variadic-macro-arguments -set(WARN_FLAGS "-Wall -Werror -Wextra -Wno-unused-parameter") +set(WARN_FLAGS "-Wall -Werror -Wextra -Wno-unused-parameter -Wno-unknown-warning-option -Wno-unused-but-set-variable") endif() if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") @@ -310,17 +316,17 @@ endif() ################################################################################################ if (BUILD_PACKAGE) - if (${CMAKE_PACKAGE_TYPE} STREQUAL "deb") + if ("${CMAKE_PACKAGE_TYPE}" STREQUAL "deb") # FIXME: hardcode it for 64-bit Linux for now set(INSTALL_LIB_DIR ${CMAKE_INSTALL_PREFIX}/lib/${CPACK_DEBIAN_ARCHITECTURE}-linux-gnu) include(tools/MakeDeb.cmake) endif() - if (${CMAKE_PACKAGE_TYPE} STREQUAL "rpm") + if ("${CMAKE_PACKAGE_TYPE}" STREQUAL "rpm") # TODO: [MG] - fix path set(INSTALL_LIB_DIR ${CMAKE_INSTALL_PREFIX}/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu) include(tools/MakeRpm.cmake) endif() - if (${CMAKE_PACKAGE_TYPE} STREQUAL "tgz") + if ("${CMAKE_PACKAGE_TYPE}" STREQUAL "tgz") # TODO: [MG] - fix path... should we simply use /usr/local/lib without CPU? # TODO: [MG] - Windows path is not ideal -- C:/Program Files (x86)/MSTelemetry/* - what should we use instead? include(tools/MakeTgz.cmake) diff --git a/build-ios.sh b/build-ios.sh index ae642750b..1fc411a7c 100755 --- a/build-ios.sh +++ b/build-ios.sh @@ -1,60 +1,90 @@ #!/bin/sh +# The expected iOS build invocation is: +# build-ios.sh [clean] [release|debug] ${ARCH} ${PLATFORM} +# where +# ARCH = arm64|arm64e|x86_64 +# PLATFORM = iphoneos|iphonesimulator|xros|xrsimulator + if [ "$1" == "clean" ]; then - rm -f CMakeCache.txt *.cmake - rm -rf out - rm -rf .buildtools -# make clean + echo "build-ios.sh: cleaning previous build artifacts" + rm -f CMakeCache.txt *.cmake + rm -rf out + rm -rf .buildtools +# make clean + shift fi -if [ "$1" == "release" ] || [ "$2" == "release" ]; then -BUILD_TYPE="Release" -else BUILD_TYPE="Debug" +if [ "$1" == "release" ]; then + BUILD_TYPE="Release" + shift +elif [ "$1" == "debug" ]; then + BUILD_TYPE="Debug" + shift fi # Set Architecture: arm64, arm64e or x86_64 -if [ "$2" == "arm64" ] || [ "$3" == "arm64" ]; then -IOS_ARCH="arm64" -elif [ "$2" == "arm64e" ] || [ "$3" == "arm64e" ]; then -IOS_ARCH="arm64e" -else -IOS_ARCH="x86_64" +IOS_ARCH=$(/usr/bin/uname -m) +if [ "$1" == "arm64" ]; then + IOS_ARCH="arm64" + shift +elif [ "$1" == "arm64e" ]; then + IOS_ARCH="arm64e" + shift +elif [ "$1" == "x86_64" ]; then + IOS_ARCH="x86_64" + shift fi -# Set Platform: device or simulator -if [ "$2" == "device" ] || [ "$3" == "device" ] || [ "$4" == "device" ]; then -IOS_PLAT="iphoneos" -else +# the last param is expected to specify the platform name: iphoneos|iphonesimulator|xros|xrsimulator +# so if it is non-empty and it is not "device", we take it as a valid platform name +# otherwise we fall back to old iOS logic which only supported iphoneos|iphonesimulator IOS_PLAT="iphonesimulator" +if [ -n "$1" ] && [ "$1" != "device" ]; then + IOS_PLAT="$1" +elif [ "$1" == "device" ]; then + IOS_PLAT="iphoneos" fi -# Set target iOS minver -default_ios_target=10.0 -if [ -z $IOS_DEPLOYMENT_TARGET ]; then - export IOS_DEPLOYMENT_TARGET=${default_ios_target} - export FORCE_RESET_DEPLOYMENT_TARGET=YES -else - export FORCE_RESET_DEPLOYMENT_TARGET=NO +echo "IOS_ARCH = $IOS_ARCH, IOS_PLAT = $IOS_PLAT, BUILD_TYPE = $BUILD_TYPE" + +FORCE_RESET_DEPLOYMENT_TARGET=NO +DEPLOYMENT_TARGET="" + +if [ "$IOS_PLAT" == "iphoneos" ] || [ "$IOS_PLAT" == "iphonesimulator" ]; then + SYS_NAME="iOS" + DEPLOYMENT_TARGET="$IOS_DEPLOYMENT_TARGET" + if [ -z "$DEPLOYMENT_TARGET" ]; then + DEPLOYMENT_TARGET="10.0" + FORCE_RESET_DEPLOYMENT_TARGET=YES + fi +elif [ "$IOS_PLAT" == "xros" ] || [ "$IOS_PLAT" == "xrsimulator" ]; then + SYS_NAME="visionOS" + DEPLOYMENT_TARGET="$XROS_DEPLOYMENT_TARGET" + if [ -z "$DEPLOYMENT_TARGET" ]; then + DEPLOYMENT_TARGET="1.0" + FORCE_RESET_DEPLOYMENT_TARGET=YES + fi fi -echo "ios deployment target="$IOS_DEPLOYMENT_TARGET -echo "force reset deployment target="$FORCE_RESET_DEPLOYMENT_TARGET + +echo "deployment target = $DEPLOYMENT_TARGET" +echo "force reset deployment target = $FORCE_RESET_DEPLOYMENT_TARGET" # Install build tools and recent sqlite3 -FILE=.buildtools -OS_NAME=`uname -a` +FILE=".buildtools" if [ ! -f $FILE ]; then tools/setup-buildtools-apple.sh ios -# Assume that the build tools have been successfully installed -echo > $FILE + # Assume that the build tools have been successfully installed + echo > $FILE fi if [ -f /usr/bin/gcc ]; then -echo "gcc version: `gcc --version`" + echo "gcc version: `gcc --version`" fi if [ -f /usr/bin/clang ]; then -echo "clang version: `clang --version`" + echo "clang version: `clang --version`" fi mkdir -p out @@ -62,8 +92,8 @@ cd out CMAKE_PACKAGE_TYPE=tgz -cmake_cmd="cmake -DCMAKE_OSX_SYSROOT=$IOS_PLAT -DCMAKE_SYSTEM_NAME=iOS -DCMAKE_IOS_ARCH_ABI=$IOS_ARCH -DCMAKE_OSX_DEPLOYMENT_TARGET=$IOS_DEPLOYMENT_TARGET -DBUILD_IOS=YES -DIOS_ARCH=$IOS_ARCH -DIOS_PLAT=$IOS_PLAT -DIOS_DEPLOYMENT_TARGET=$IOS_DEPLOYMENT_TARGET -DBUILD_UNIT_TESTS=YES -DBUILD_FUNC_TESTS=YES -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_PACKAGE_TYPE=$CMAKE_PACKAGE_TYPE -DFORCE_RESET_DEPLOYMENT_TARGET=$FORCE_RESET_DEPLOYMENT_TARGET $CMAKE_OPTS .." -echo $cmake_cmd +cmake_cmd="cmake -DCMAKE_OSX_SYSROOT=$IOS_PLAT -DCMAKE_SYSTEM_NAME=$SYS_NAME -DCMAKE_IOS_ARCH_ABI=$IOS_ARCH -DCMAKE_OSX_DEPLOYMENT_TARGET=$DEPLOYMENT_TARGET -DBUILD_IOS=YES -DIOS_ARCH=$IOS_ARCH -DIOS_PLAT=$IOS_PLAT -DIOS_DEPLOYMENT_TARGET=$DEPLOYMENT_TARGET -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_PACKAGE_TYPE=$CMAKE_PACKAGE_TYPE -DFORCE_RESET_DEPLOYMENT_TARGET=$FORCE_RESET_DEPLOYMENT_TARGET $CMAKE_OPTS .." +echo "${cmake_cmd}" eval $cmake_cmd make diff --git a/build.sh b/build.sh index f3c7909b0..ee864d938 100755 --- a/build.sh +++ b/build.sh @@ -1,5 +1,32 @@ #!/bin/bash +usage() +{ + echo "Usage: build.sh [clean] [arm64|x86_64|universal] [CUSTOM_CMAKE_CXX_FLAGS=x] [noroot] [release] [-h|-?] [-l (static|shared)] [-D CMAKE_OPTION] [-v]" + echo " " + echo "options: " + echo " " + echo "Positional options (1st three arguments): " + echo "[clean] - perform clean build " + echo "[arm64|x86_64|universal] - Apple platform build type. Not applicable to other OS. " + echo "[CUSTOM_CMAKE_CXX_FLAGS] - custom CXX compiler flags " + echo "[noroot] - custom CXX compiler flags " + echo "[release] - build for Release " + echo " " + echo "Additional parameters: " + echo " -h | -? - this help. " + echo " -l [static|shared] - build static (default) or shared library. " + echo " -D [CMAKE_OPTION] - additional options to pass to cmake. Could be multiple. " + echo " -v - increase build verbosity (reserved for future use) " + echo " " + echo "Environment variables: " + echo "CMAKE_OPTS - any additional cmake options. " + echo "GIT_PULL_TOKEN - authorization token for Microsoft-proprietary modules. " + echo "MACOSX_DEPLOYMENT_TARGET - optional parameter for setting macosx deployment target " + echo "Plus any other environment variables respected by CMake build system. " + exit 0 +} + export PATH=/usr/local/bin:$PATH DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" @@ -8,78 +35,80 @@ cd $DIR export NOROOT=$NOROOT -if [ "$1" == "clean" ]; then +PARAM1="$1" +PARAM2="$2" +PARAM3="$3" + +if [ "$PARAM1" == "clean" ]; then + echo "Cleaning previous build artifacts" rm -f CMakeCache.txt *.cmake rm -rf out rm -rf .buildtools # make clean + shift fi -if [ "$1" == "noroot" ] || [ "$2" == "noroot" ] || [ "$3" == "noroot" ]; then +if [ "$PARAM1" == "noroot" ] || [ "$PARAM2" == "noroot" ] || [ "$PARAM3" == "noroot" ]; then export NOROOT=true + echo "NOROOT = true" + shift fi -if [ "$1" == "release" ] || [ "$2" == "release" ] || [ "$3" == "release" ]; then +if [ "$PARAM1" == "release" ] || [ "$PARAM2" == "release" ] || [ "$PARAM3" == "release" ]; then BUILD_TYPE="Release" + echo "BUILD_TYPE = Release" + shift +elif [ "$PARAM1" == "debug" ] || [ "$PARAM2" == "debug" ] || [ "$PARAM3" == "debug" ]; then + BUILD_TYPE="Debug" + echo "BUILD_TYPE = Debug" + shift else BUILD_TYPE="Debug" + echo "Assuming default BUILD_TYPE = Debug" fi -if [ "$1" == "arm64" ] || [ "$2" == "arm64" ] || [ "$3" == "arm64" ]; then +if [ "$PARAM1" == "arm64" ] || [ "$PARAM2" == "arm64" ] || [ "$PARAM3" == "arm64" ]; then MAC_ARCH="arm64" -elif [ "$1" == "universal" ] || [ "$2" == "universal" ] || [ "$3" == "universal" ]; then + echo "MAC_ARCH = arm64" + shift +elif [ "$PARAM1" == "universal" ] || [ "$PARAM2" == "universal" ] || [ "$PARAM3" == "universal" ]; then MAC_ARCH="universal" -else + echo "MAC_ARCH = universal" + shift +elif [ "$PARAM1" == "x86_64" ] || [ "$PARAM2" == "x86_64" ] || [ "$PARAM3" == "x86_64" ]; then MAC_ARCH="x86_64" + echo "MAC_ARCH = x86_64" + shift +else + MAC_ARCH=$(/usr/bin/uname -m) + echo "Using current machine MAC_ARCH = $MAC_ARCH" fi CUSTOM_CMAKE_CXX_FLAG="" -if [[ $1 == CUSTOM_BUILD_FLAGS* ]] || [[ $2 == CUSTOM_BUILD_FLAGS* ]] || [[ $3 == CUSTOM_BUILD_FLAGS* ]]; then - if [[ $1 == CUSTOM_BUILD_FLAGS* ]]; then +if [[ $PARAM1 == CUSTOM_BUILD_FLAGS* ]] || [[ $PARAM2 == CUSTOM_BUILD_FLAGS* ]] || [[ $PARAM3 == CUSTOM_BUILD_FLAGS* ]]; then + if [[ $PARAM1 == CUSTOM_BUILD_FLAGS* ]]; then CUSTOM_CMAKE_CXX_FLAG="\"${1:19:999}\"" - elif [[ $2 == CUSTOM_BUILD_FLAGS* ]]; then + elif [[ $PARAM2 == CUSTOM_BUILD_FLAGS* ]]; then CUSTOM_CMAKE_CXX_FLAG="\"${2:19:999}\"" - elif [[ $3 == CUSTOM_BUILD_FLAGS* ]]; then + elif [[ $PARAM3 == CUSTOM_BUILD_FLAGS* ]]; then CUSTOM_CMAKE_CXX_FLAG="\"${3:19:999}\"" fi - echo "custom build flags="$CUSTOM_CMAKE_CXX_FLAG + shift + echo "custom build flags = $CUSTOM_CMAKE_CXX_FLAG" fi LINK_TYPE= CMAKE_OPTS="${CMAKE_OPTS:--DBUILD_SHARED_LIBS=OFF}" while getopts "h?vl:D:" opt; do case "$opt" in - h|\?) - echo "Usage: build.sh [clean] [arm64|universal] [CUSTOM_CMAKE_CXX_FLAGS=x] [noroot] [release] [-h|-?] [-l (static|shared)] [-D CMAKE_OPTION] [-v]" - echo " " - echo "options: " - echo " " - echo "Positional options (1st three arguments): " - echo "[clean] - perform clean build " - echo "[arm64|universal] - Apple platform build type. Not applicable to other OS. " - echo "[CUSTOM_CMAKE_CXX_FLAGS] - custom CXX compiler flags " - echo "[noroot] - custom CXX compiler flags " - echo "[release] - build for Release " - echo " " - echo "Additional parameters: " - echo " -h | -? - this help. " - echo " -l [static|shared] - build static (default) or shared library. " - echo " -D [CMAKE_OPTION] - additional options to pass to cmake. Could be multiple. " - echo " -v - increase build verbosity (reserved for future use) " - echo " " - echo "Environment variables: " - echo "CMAKE_OPTS - any additional cmake options. " - echo "GIT_PULL_TOKEN - authorization token for Microsoft-proprietary modules. " - echo "MACOSX_DEPLOYMENT_TARGET - optional parameter for setting macosx deployment target " - echo "Plus any other environment variables respected by CMake build system. " - exit 0 + h|\?) usage ;; :) echo "Invalid option: $OPTARG requires an argument" 1>&2 exit 0 ;; v) verbose=1 ;; - D) CMAKE_OPTS="$CMAKE_OPTS -D$OPTARG" + D) CMAKE_OPTS="${CMAKE_OPTS} -D${OPTARG}" ;; l) LINK_TYPE=$OPTARG ;; @@ -87,8 +116,10 @@ while getopts "h?vl:D:" opt; do done shift $((OPTIND -1)) +echo "CMAKE_OPTS from caller: $CMAKE_OPTS" + if [ "$LINK_TYPE" == "shared" ]; then - CMAKE_OPTS="$CMAKE_OPTS -DBUILD_SHARED_LIBS=ON" + CMAKE_OPTS="${CMAKE_OPTS} -DBUILD_SHARED_LIBS=ON" fi # Set target MacOS minver