Merge branch 'main' into morewebhooks

.github/workflows/publish.yml

@@ -15,6 +15,10 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
       - name: Log in to the Container registry
         uses: docker/login-action@v2
         with:
@@ -30,6 +34,7 @@ jobs:
         uses: docker/build-push-action@v3
         with:
           context: .
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}

Dockerfile

@@ -1,5 +1,5 @@
 # Build image
-FROM golang:1.20-alpine as build
+FROM golang:1.21-alpine as build
 
 WORKDIR /go/src/app
 COPY . .

README.md

@@ -12,7 +12,7 @@ At this time this is likely to be mostly development discussion.
 
 # Requirements
 
-[go 1.20](https://go.dev/doc/install)
+[go 1.21](https://go.dev/doc/install)
 
 # Instructions
 
@@ -112,7 +112,13 @@ innodb_io_capacity_max=2000
 
 # Trust disk system at the expense of recovery
 innodb_doublewrite = 0
+```
+
+These last two options prevent your ibdata1 file from growing continuously
+and space not being re-used.  If you have this problem you can add these but
+be aware this requires recreation of ibdata.
 
+```toml
 # Efficiently use ibdata
 innodb_undo_log_truncate = 1
 innodb_undo_tablespaces = 4

config.toml.example

@@ -1,9 +1,9 @@
 port = 9001             # Listening port for golbat
+#grpc_port = 50001       # Listening port for grpc
 raw_bearer = ""         # Raw bearer (password) required
 api_secret = "golbat"   # Golbat secret required on api calls (blank for none)
 
 pokemon_memory_only = false  # Use in-memory storage for pokemon only
-stats = false           # Enable/Disable pokemon stats history (this should normally be disabled - real time stats are calculated always)
 
 [koji]
 url = "http://{koji_url}/api/v1/geofence/feature-collection/{golbat_project}"
@@ -38,7 +38,7 @@ include_hundos_under_cap = false
 [[webhooks]]
 url = "http://localhost:4201"
 # types if specified can be...
-# types = ["pokemon", "gym", "invasion", "quest", "pokestop", "raid"]
+# types = ["pokemon", "gym", "invasion", "quest", "pokestop", "raid", "weather", "fort_update"]
 
 #[[webhooks]]
 #url = "http://localhost:4202"
@@ -47,4 +47,4 @@ url = "http://localhost:4201"
 #[[webhooks]]
 #url = "http://localhost:4202"
 #types = ["raid"]
-#areas = ["London/*", "*/Harrow", "Harrow"]
+#areas = ["London/*", "*/Harrow", "Harrow"]

config/config.go

@@ -4,9 +4,9 @@ import "golbat/geo"
 
 type configDefinition struct {
 	Port              int        `koanf:"port"`
+	GrpcPort          int        `koanf:"grpc_port"`
 	Webhooks          []webhook  `koanf:"webhooks"`
 	Database          database   `koanf:"database"`
-	Stats             bool       `koanf:"stats"`
 	Logging           logging    `koanf:"logging"`
 	Sentry            sentry     `koanf:"sentry"`
 	Pyroscope         pyroscope  `koanf:"pyroscope"`

decoder/fort.go

@@ -8,6 +8,7 @@ import (
 	"golbat/pogo"
 	"golbat/webhooks"
 	"net/url"
+	"strings"
 )
 
 type Location struct {
@@ -141,31 +142,36 @@ func CreateFortWebHooks(old *FortWebhook, new *FortWebhook, change FortChange) {
 	} else if change == EDIT {
 		areas := MatchStatsGeofence(new.Location.Latitude, new.Location.Longitude)
 		var editTypes []string
-		if !(old.Name == nil && new.Name == nil) &&
-			(old.Name == nil || new.Name == nil || *old.Name != *new.Name) {
+
+		// Check if Name has changed
+		if old.Name == nil {
+			if new.Name != nil && *new.Name != "" {
+				editTypes = append(editTypes, "name")
+			}
+		} else if new.Name != nil && *old.Name != *new.Name {
 			editTypes = append(editTypes, "name")
 		}
-		if !(old.Description == nil && new.Description == nil) &&
-			(old.Description == nil || new.Description == nil || *old.Description != *new.Description) {
+
+		// Check if Description has changed
+		if old.Description == nil {
+			if new.Description != nil && *new.Description != "" {
+				editTypes = append(editTypes, "description")
+			}
+		} else if new.Description != nil && *old.Description != *new.Description {
 			editTypes = append(editTypes, "description")
 		}
-		if !(old.ImageUrl == nil && new.ImageUrl == nil) &&
-			(old.ImageUrl == nil || new.ImageUrl == nil || *old.ImageUrl != *new.ImageUrl) {
-			var newPath, oldPath string
-			newUrl, err := url.Parse(*new.ImageUrl)
-			if err == nil && newUrl != nil {
-				newPath = newUrl.Path
-			}
-			if old.ImageUrl != nil {
-				oldUrl, err2 := url.Parse(*old.ImageUrl)
-				if err2 == nil && oldUrl != nil {
-					oldPath = oldUrl.Path
-				}
-			}
+
+		// Check if ImageUrl has changed
+		if old.ImageUrl != nil && new.ImageUrl != nil && *old.ImageUrl != *new.ImageUrl {
+			oldPath := getPathFromURL(*old.ImageUrl)
+			newPath := getPathFromURL(*new.ImageUrl)
 			if oldPath != newPath {
 				editTypes = append(editTypes, "image_url")
 			}
+		} else if (old.ImageUrl == nil || *old.ImageUrl == "") && new.ImageUrl != nil && *new.ImageUrl != "" {
+			editTypes = append(editTypes, "image_url")
 		}
+		// Check if location has changed
 		if !floatAlmostEqual(old.Location.Latitude, new.Location.Latitude, floatTolerance) ||
 			!floatAlmostEqual(old.Location.Longitude, new.Location.Longitude, floatTolerance) {
 			editTypes = append(editTypes, "location")
@@ -182,6 +188,13 @@ func CreateFortWebHooks(old *FortWebhook, new *FortWebhook, change FortChange) {
 	}
 }
 
+func getPathFromURL(u string) string {
+	parsedURL, err := url.Parse(u)
+	if err != nil {
+		return ""
+	}
+	return strings.TrimPrefix(parsedURL.Path, "/")
+}
 func UpdateFortRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails, mapFort *pogo.GetMapFortsOutProto_FortProto) (bool, string) {
 	// when we miss, we check the gym, if again, we save it in cache for 5 minutes (in gym part)
 	status, output := UpdatePokestopRecordWithGetMapFortsOutProto(ctx, db, mapFort)

decoder/gym.go

@@ -261,7 +261,12 @@ func (gym *Gym) updateGymFromGymInfoOutProto(gymData *pogo.GymGetInfoOutProto) *
 		gym.Url = null.StringFrom(gymData.Url)
 	}
 	gym.Name = null.StringFrom(gymData.Name)
-	gym.Description = null.StringFrom(gymData.Description)
+
+	if gymData.Description == "" {
+		gym.Description = null.NewString("", false)
+	} else {
+		gym.Description = null.StringFrom(gymData.Description)
+	}
 
 	return gym
 }

decoder/main.go

@@ -52,6 +52,7 @@ var spawnpointCache *ttlcache.Cache[int64, Spawnpoint]
 var pokemonCache *ttlcache.Cache[string, Pokemon]
 var incidentCache *ttlcache.Cache[string, Incident]
 var playerCache *ttlcache.Cache[string, Player]
+var routeCache *ttlcache.Cache[string, Route]
 var diskEncounterCache *ttlcache.Cache[string, *pogo.DiskEncounterOutProto]
 var getMapFortsCache *ttlcache.Cache[string, *pogo.GetMapFortsOutProto_FortProto]
 
@@ -61,6 +62,7 @@ var incidentStripedMutex = stripedmutex.New(128)
 var pokemonStripedMutex = stripedmutex.New(1024)
 var weatherStripedMutex = stripedmutex.New(128)
 var s2cellStripedMutex = stripedmutex.New(1024)
+var routeStripedMutex = stripedmutex.New(128)
 
 var s2CellLookup = sync.Map{}
 
@@ -132,6 +134,11 @@ func initDataCache() {
 		ttlcache.WithDisableTouchOnHit[string, *pogo.GetMapFortsOutProto_FortProto](),
 	)
 	go getMapFortsCache.Start()
+
+	routeCache = ttlcache.New[string, Route](
+		ttlcache.WithTTL[string, Route](60 * time.Minute),
+	)
+	go routeCache.Start()
 }
 
 func InitialiseOhbem() {

decoder/pokemon.go

@@ -310,6 +310,7 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po
 			if err != nil {
 				log.Errorf("insert pokemon: [%s] %s", pokemon.Id, err)
 				log.Errorf("Full structure: %+v", pokemon)
+				pokemonCache.Delete(pokemon.Id) // Force reload of pokemon from database
 				return
 			}
 
@@ -360,12 +361,12 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po
 			if err != nil {
 				log.Errorf("Update pokemon [%s] %s", pokemon.Id, err)
 				log.Errorf("Full structure: %+v", pokemon)
+				pokemonCache.Delete(pokemon.Id) // Force reload of pokemon from database
+
 				return
 			}
 			rows, rowsErr := res.RowsAffected()
 			log.Debugf("Updating pokemon [%s] after update res = %d %v", pokemon.Id, rows, rowsErr)
-
-			_, _ = res, err
 		}
 	}
 
@@ -501,15 +502,11 @@ func (pokemon *Pokemon) wildSignificantUpdate(wildPokemon *pogo.WildPokemonProto
 
 func (pokemon *Pokemon) updateFromWild(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, cellId int64, timestampMs int64, username string) {
 	pokemon.IsEvent = 0
-	encounterId := strconv.FormatUint(wildPokemon.EncounterId, 10)
 	switch pokemon.SeenType.ValueOrZero() {
 	case "", SeenType_Cell, SeenType_NearbyStop:
 		pokemon.SeenType = null.StringFrom(SeenType_Wild)
-		updateStats(ctx, db, encounterId, stats_seenWild)
-	}
-	if pokemon.addWildPokemon(ctx, db, wildPokemon, timestampMs) {
-		updateStats(ctx, db, pokemon.Id, stats_statsReset)
 	}
+	_ = pokemon.addWildPokemon(ctx, db, wildPokemon, timestampMs)
 	pokemon.repopulateStatsIfNeeded(ctx, db)
 	pokemon.Username = null.StringFrom(username)
 	pokemon.CellId = null.IntFrom(cellId)
@@ -526,7 +523,6 @@ func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapP
 
 	encounterId := strconv.FormatUint(mapPokemon.EncounterId, 10)
 	pokemon.Id = encounterId
-	updateStats(ctx, db, encounterId, stats_seenLure)
 
 	spawnpointId := mapPokemon.SpawnpointId
 
@@ -542,9 +538,7 @@ func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapP
 	pokemon.SeenType = null.StringFrom(SeenType_LureWild) // TODO may have been encounter... this needs fixing
 
 	if mapPokemon.PokemonDisplay != nil {
-		if pokemon.setPokemonDisplay(pokemon.PokemonId, mapPokemon.PokemonDisplay) {
-			updateStats(ctx, db, pokemon.Id, stats_statsReset)
-		}
+		_ = pokemon.setPokemonDisplay(pokemon.PokemonId, mapPokemon.PokemonDisplay)
 		pokemon.repopulateStatsIfNeeded(ctx, db)
 		// The mapPokemon and nearbyPokemon GMOs don't contain actual shininess.
 		// shiny = mapPokemon.pokemonDisplay.shiny
@@ -574,23 +568,12 @@ func (pokemon *Pokemon) calculateIv(a int64, d int64, s int64) {
 
 func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, nearbyPokemon *pogo.NearbyPokemonProto, cellId int64, username string) {
 	pokemon.IsEvent = 0
-	encounterId := strconv.FormatUint(nearbyPokemon.EncounterId, 10)
 	pokestopId := nearbyPokemon.FortId
 	pokemonId := int16(nearbyPokemon.PokedexNumber)
-	if pokemon.setPokemonDisplay(pokemonId, nearbyPokemon.PokemonDisplay) {
-		updateStats(ctx, db, pokemon.Id, stats_statsReset)
-	}
+	_ = pokemon.setPokemonDisplay(pokemonId, nearbyPokemon.PokemonDisplay)
 	pokemon.repopulateStatsIfNeeded(ctx, db)
 	pokemon.Username = null.StringFrom(username)
 
-	if pokemon.isNewRecord() {
-		if pokestopId == "" {
-			updateStats(ctx, db, encounterId, stats_seenCell)
-		} else {
-			updateStats(ctx, db, encounterId, stats_seenStop)
-		}
-	}
-
 	var lat, lon float64
 	overrideLatLon := pokemon.isNewRecord()
 	useCellLatLon := true
@@ -939,7 +922,6 @@ func (pokemon *Pokemon) updatePokemonFromEncounterProto(ctx context.Context, db
 	pokemon.Username = null.StringFrom(username)
 
 	pokemon.SeenType = null.StringFrom(SeenType_Encounter)
-	updateStats(ctx, db, pokemon.Id, stats_encounter)
 }
 
 func (pokemon *Pokemon) updatePokemonFromDiskEncounterProto(ctx context.Context, db db.DbDetails, encounterData *pogo.DiskEncounterOutProto) {
@@ -959,7 +941,6 @@ func (pokemon *Pokemon) updatePokemonFromDiskEncounterProto(ctx context.Context,
 	}
 
 	pokemon.SeenType = null.StringFrom(SeenType_LureEncounter)
-	updateStats(ctx, db, pokemon.Id, stats_lureEncounter)
 }
 
 func (pokemon *Pokemon) setPokemonDisplay(pokemonId int16, display *pogo.PokemonDisplayProto) bool {
@@ -1166,41 +1147,3 @@ func UpdatePokemonRecordWithDiskEncounterProto(ctx context.Context, db db.DbDeta
 
 	return fmt.Sprintf("%s Disk Pokemon %d CP%d", encounterId, pokemon.PokemonId, encounter.Pokemon.Cp)
 }
-
-const stats_seenWild string = "seen_wild"
-const stats_seenStop string = "seen_stop"
-const stats_seenCell string = "seen_cell"
-const stats_statsReset string = "stats_reset"
-const stats_encounter string = "encounter"
-const stats_seenLure string = "seen_lure"
-const stats_lureEncounter string = "lure_encounter"
-
-func updateStats(ctx context.Context, db db.DbDetails, id string, event string) {
-	if config.Config.Stats == false {
-		return
-	}
-
-	var sqlCommand string
-
-	if event == stats_encounter {
-		sqlCommand = "INSERT INTO pokemon_timing (id, seen_wild, first_encounter, last_encounter) " +
-			"VALUES (?, UNIX_TIMESTAMP(), UNIX_TIMESTAMP(), UNIX_TIMESTAMP()) " +
-			"ON DUPLICATE KEY UPDATE " +
-			"first_encounter = COALESCE(first_encounter, VALUES(first_encounter))," +
-			"last_encounter = VALUES(last_encounter)," +
-			"seen_wild = COALESCE(seen_wild, first_encounter)"
-	} else {
-		sqlCommand = fmt.Sprintf("INSERT INTO pokemon_timing (id, %[1]s)"+
-			"VALUES (?, UNIX_TIMESTAMP())"+
-			"ON DUPLICATE KEY UPDATE "+
-			"%[1]s=COALESCE(%[1]s, VALUES(%[1]s))", event)
-	}
-
-	log.Debugf("Updating pokemon timing: [%s] %s", id, event)
-	_, err := db.GeneralDb.ExecContext(ctx, sqlCommand, id)
-
-	if err != nil {
-		log.Errorf("update pokemon timing: [%s] %s", id, err)
-		return
-	}
-}