Skip to content

Commit

Permalink
Remove Legacy Stats (#143)
Browse files Browse the repository at this point in the history 
* Remove legacy stats references

* Clean example

* Remove legacy references

* clean database

* rename migration
  • Loading branch information
@jfberry
jfberry authored Aug 19, 2023
1 parent 813a9c5 commit 198b5c5
Show file tree
Hide file tree
Showing 7 changed files with 74 additions and 206 deletions.
1 change: 0 additions & 1 deletion config.toml.example
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ raw_bearer = "" # Raw bearer (password) required
api_secret = "golbat" # Golbat secret required on api calls (blank for none)

pokemon_memory_only = false # Use in-memory storage for pokemon only
stats = false # Enable/Disable pokemon stats history (this should normally be disabled - real time stats are calculated always)

[koji]
url = "http://{koji_url}/api/v1/geofence/feature-collection/{golbat_project}"
Expand Down
1 change: 0 additions & 1 deletion config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ type configDefinition struct {
GrpcPort int `koanf:"grpc_port"`
Webhooks []webhook `koanf:"webhooks"`
Database database `koanf:"database"`
Stats bool `koanf:"stats"`
Logging logging `koanf:"logging"`
Sentry sentry `koanf:"sentry"`
Pyroscope pyroscope `koanf:"pyroscope"`
Expand Down
64 changes: 3 additions & 61 deletions decoder/pokemon.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -507,15 +507,11 @@ func (pokemon *Pokemon) wildSignificantUpdate(wildPokemon *pogo.WildPokemonProto

func (pokemon *Pokemon) updateFromWild(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, cellId int64, timestampMs int64, username string) {
pokemon.IsEvent = 0
encounterId := strconv.FormatUint(wildPokemon.EncounterId, 10)
switch pokemon.SeenType.ValueOrZero() {
case "", SeenType_Cell, SeenType_NearbyStop:
pokemon.SeenType = null.StringFrom(SeenType_Wild)
updateStats(ctx, db, encounterId, stats_seenWild)
}
if pokemon.addWildPokemon(ctx, db, wildPokemon, timestampMs) {
updateStats(ctx, db, pokemon.Id, stats_statsReset)
}
_ = pokemon.addWildPokemon(ctx, db, wildPokemon, timestampMs)
pokemon.repopulateStatsIfNeeded(ctx, db)
pokemon.Username = null.StringFrom(username)
pokemon.CellId = null.IntFrom(cellId)
Expand All @@ -532,7 +528,6 @@ func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapP

encounterId := strconv.FormatUint(mapPokemon.EncounterId, 10)
pokemon.Id = encounterId
updateStats(ctx, db, encounterId, stats_seenLure)

spawnpointId := mapPokemon.SpawnpointId

Expand All @@ -548,9 +543,7 @@ func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapP
pokemon.SeenType = null.StringFrom(SeenType_LureWild) // TODO may have been encounter... this needs fixing

if mapPokemon.PokemonDisplay != nil {
if pokemon.setPokemonDisplay(pokemon.PokemonId, mapPokemon.PokemonDisplay) {
updateStats(ctx, db, pokemon.Id, stats_statsReset)
}
_ = pokemon.setPokemonDisplay(pokemon.PokemonId, mapPokemon.PokemonDisplay)
pokemon.repopulateStatsIfNeeded(ctx, db)
// The mapPokemon and nearbyPokemon GMOs don't contain actual shininess.
// shiny = mapPokemon.pokemonDisplay.shiny
Expand Down Expand Up @@ -580,23 +573,12 @@ func (pokemon *Pokemon) calculateIv(a int64, d int64, s int64) {

func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, nearbyPokemon *pogo.NearbyPokemonProto, cellId int64, username string) {
pokemon.IsEvent = 0
encounterId := strconv.FormatUint(nearbyPokemon.EncounterId, 10)
pokestopId := nearbyPokemon.FortId
pokemonId := int16(nearbyPokemon.PokedexNumber)
if pokemon.setPokemonDisplay(pokemonId, nearbyPokemon.PokemonDisplay) {
updateStats(ctx, db, pokemon.Id, stats_statsReset)
}
_ = pokemon.setPokemonDisplay(pokemonId, nearbyPokemon.PokemonDisplay)
pokemon.repopulateStatsIfNeeded(ctx, db)
pokemon.Username = null.StringFrom(username)

if pokemon.isNewRecord() {
if pokestopId == "" {
updateStats(ctx, db, encounterId, stats_seenCell)
} else {
updateStats(ctx, db, encounterId, stats_seenStop)
}
}

var lat, lon float64
overrideLatLon := pokemon.isNewRecord()
useCellLatLon := true
Expand Down Expand Up @@ -945,7 +927,6 @@ func (pokemon *Pokemon) updatePokemonFromEncounterProto(ctx context.Context, db
pokemon.Username = null.StringFrom(username)

pokemon.SeenType = null.StringFrom(SeenType_Encounter)
updateStats(ctx, db, pokemon.Id, stats_encounter)
}

func (pokemon *Pokemon) updatePokemonFromDiskEncounterProto(ctx context.Context, db db.DbDetails, encounterData *pogo.DiskEncounterOutProto) {
Expand All @@ -965,7 +946,6 @@ func (pokemon *Pokemon) updatePokemonFromDiskEncounterProto(ctx context.Context,
}

pokemon.SeenType = null.StringFrom(SeenType_LureEncounter)
updateStats(ctx, db, pokemon.Id, stats_lureEncounter)
}

func (pokemon *Pokemon) setPokemonDisplay(pokemonId int16, display *pogo.PokemonDisplayProto) bool {
Expand Down Expand Up @@ -1172,41 +1152,3 @@ func UpdatePokemonRecordWithDiskEncounterProto(ctx context.Context, db db.DbDeta

return fmt.Sprintf("%s Disk Pokemon %d CP%d", encounterId, pokemon.PokemonId, encounter.Pokemon.Cp)
}

const stats_seenWild string = "seen_wild"
const stats_seenStop string = "seen_stop"
const stats_seenCell string = "seen_cell"
const stats_statsReset string = "stats_reset"
const stats_encounter string = "encounter"
const stats_seenLure string = "seen_lure"
const stats_lureEncounter string = "lure_encounter"

func updateStats(ctx context.Context, db db.DbDetails, id string, event string) {
if config.Config.Stats == false {
return
}

var sqlCommand string

if event == stats_encounter {
sqlCommand = "INSERT INTO pokemon_timing (id, seen_wild, first_encounter, last_encounter) " +
"VALUES (?, UNIX_TIMESTAMP(), UNIX_TIMESTAMP(), UNIX_TIMESTAMP()) " +
"ON DUPLICATE KEY UPDATE " +
"first_encounter = COALESCE(first_encounter, VALUES(first_encounter))," +
"last_encounter = VALUES(last_encounter)," +
"seen_wild = COALESCE(seen_wild, first_encounter)"
} else {
sqlCommand = fmt.Sprintf("INSERT INTO pokemon_timing (id, %[1]s)"+
"VALUES (?, UNIX_TIMESTAMP())"+
"ON DUPLICATE KEY UPDATE "+
"%[1]s=COALESCE(%[1]s, VALUES(%[1]s))", event)
}

log.Debugf("Updating pokemon timing: [%s] %s", id, event)
_, err := db.GeneralDb.ExecContext(ctx, sqlCommand, id)

if err != nil {
log.Errorf("update pokemon timing: [%s] %s", id, err)
return
}
}
1 change: 0 additions & 1 deletion main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import (
)

var db *sqlx.DB
var inMemoryDb *sqlx.DB
var dbDetails db2.DbDetails

func main() {
Expand Down
2 changes: 2 additions & 0 deletions sql/22_cleanup.up.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
drop procedure if exists createStatsAndArchive;
drop table if exists pokemon_timing;
23 changes: 1 addition & 22 deletions sql/2_stats_tables.up.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,28 +48,7 @@ CREATE TABLE `pokemon_timing` (
create procedure createStatsAndArchive()
begin
drop temporary table if exists old;
create temporary table old engine = memory
as (select id from pokemon where expire_timestamp < (UNIX_TIMESTAMP() - 3600));

insert into pokemon_history (id, location, pokemon_id, cp, atk_iv, def_iv, sta_iv, form, level, weather,
costume, cell_id, expire_timestamp, expire_timestamp_verified, display_pokemon_id,
seen_type, shiny, seen_wild, seen_stop, seen_cell, seen_lure,
first_encounter, stats_reset, last_encounter, lure_encounter)
select pokemon.id, POINT(lat,lon) as location, pokemon_id, cp, atk_iv, def_iv, sta_iv, form, level, weather,
costume, cell_id, expire_timestamp, expire_timestamp_verified, display_pokemon_id,
seen_type, shiny, seen_wild, seen_stop, seen_cell, seen_lure,
first_encounter, stats_reset, last_encounter, lure_encounter
from pokemon
join old on old.id = pokemon.id
left join pokemon_timing on pokemon.id = pokemon_timing.id;

delete pokemon from pokemon
join old on pokemon.id = old.id;

delete pokemon_timing from pokemon_timing
join old on pokemon_timing.id = old.id;

drop temporary table old;
/* removed */
end;


188 changes: 68 additions & 120 deletions stats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,121 +31,99 @@ const databaseDeleteChunkSize = 500
func StartDatabaseArchiver(db *sqlx.DB) {
ticker := time.NewTicker(time.Minute)

if config.Config.Stats {
go func() {
for {
<-ticker.C
start := time.Now()

var result sql.Result
var err error
go func() {
for {
<-ticker.C
log.Infof("DB - Archive of pokemon table - starting")
start := time.Now()

result, err = db.Exec("call createStatsAndArchive();")
var resultCounter int64
var result sql.Result
var err error

elapsed := time.Since(start)
start = time.Now()

for {
pokemonId := []PokemonIdToDelete{}
err = db.Select(&pokemonId,
fmt.Sprintf("SELECT id FROM pokemon WHERE expire_timestamp < UNIX_TIMESTAMP() AND expire_timestamp_verified = 1 LIMIT %d;", databaseDeleteChunkSize))
if err != nil {
log.Errorf("DB - Archive of pokemon table error %s", err)
} else {
rows, _ := result.RowsAffected()
log.Infof("DB - Archive of pokemon table took %s (%d rows)", elapsed, rows)
log.Errorf("DB - Archive of pokemon table (expire time verified) select error [after %d rows] %s", resultCounter, err)
break
}
}
}()
} else {
go func() {
for {
<-ticker.C
log.Infof("DB - Archive of pokemon table - starting")
start := time.Now()

var resultCounter int64
var result sql.Result
var err error

start = time.Now()

for {
pokemonId := []PokemonIdToDelete{}
err = db.Select(&pokemonId,
fmt.Sprintf("SELECT id FROM pokemon WHERE expire_timestamp < UNIX_TIMESTAMP() AND expire_timestamp_verified = 1 LIMIT %d;", databaseDeleteChunkSize))
if err != nil {
log.Errorf("DB - Archive of pokemon table (expire time verified) select error [after %d rows] %s", resultCounter, err)
break
}

if len(pokemonId) == 0 {
break
}
if len(pokemonId) == 0 {
break
}

var ids []string
for i := 0; i < len(pokemonId); i++ {
ids = append(ids, pokemonId[i].Id)
}
var ids []string
for i := 0; i < len(pokemonId); i++ {
ids = append(ids, pokemonId[i].Id)
}

query, args, _ := sqlx.In("DELETE FROM pokemon WHERE id IN (?);", ids)
query = db.Rebind(query)
query, args, _ := sqlx.In("DELETE FROM pokemon WHERE id IN (?);", ids)
query = db.Rebind(query)

result, err = db.Exec(query, args...)
result, err = db.Exec(query, args...)

if err != nil {
log.Errorf("DB - Archive of pokemon table (expire time verified) error [after %d rows] %s", resultCounter, err)
if err != nil {
log.Errorf("DB - Archive of pokemon table (expire time verified) error [after %d rows] %s", resultCounter, err)
break
} else {
rows, _ := result.RowsAffected()
resultCounter += rows
if rows < databaseDeleteChunkSize {
elapsed := time.Since(start)
log.Infof("DB - Archive of pokemon table (verified timestamps) took %s (%d rows)", elapsed, resultCounter)
break
} else {
rows, _ := result.RowsAffected()
resultCounter += rows
if rows < databaseDeleteChunkSize {
elapsed := time.Since(start)
log.Infof("DB - Archive of pokemon table (verified timestamps) took %s (%d rows)", elapsed, resultCounter)
break
}
}
}
}

log.Infof("DB - Archive of pokemon table - starting second phase (unverified timestamps)")
log.Infof("DB - Archive of pokemon table - starting second phase (unverified timestamps)")

resultCounter = 0
start = time.Now()
resultCounter = 0
start = time.Now()

for {
pokemonId := []PokemonIdToDelete{}
err = db.Select(&pokemonId,
fmt.Sprintf("SELECT id FROM pokemon WHERE expire_timestamp < (UNIX_TIMESTAMP() - 2400) AND expire_timestamp_verified = 0 LIMIT %d;", databaseDeleteChunkSize))
if err != nil {
log.Errorf("DB - Archive of pokemon table (unverified timestamps) select error [after %d rows] %s", resultCounter, err)
break
}
for {
pokemonId := []PokemonIdToDelete{}
err = db.Select(&pokemonId,
fmt.Sprintf("SELECT id FROM pokemon WHERE expire_timestamp < (UNIX_TIMESTAMP() - 2400) AND expire_timestamp_verified = 0 LIMIT %d;", databaseDeleteChunkSize))
if err != nil {
log.Errorf("DB - Archive of pokemon table (unverified timestamps) select error [after %d rows] %s", resultCounter, err)
break
}

if len(pokemonId) == 0 {
break
}
if len(pokemonId) == 0 {
break
}

var ids []string
for i := 0; i < len(pokemonId); i++ {
ids = append(ids, pokemonId[i].Id)
}
var ids []string
for i := 0; i < len(pokemonId); i++ {
ids = append(ids, pokemonId[i].Id)
}

query, args, _ := sqlx.In("DELETE FROM pokemon WHERE id IN (?);", ids)
query = db.Rebind(query)
query, args, _ := sqlx.In("DELETE FROM pokemon WHERE id IN (?);", ids)
query = db.Rebind(query)

result, err = db.Exec(query, args...)
result, err = db.Exec(query, args...)

if err != nil {
log.Errorf("DB - Archive of pokemon table (unverified timestamps) error [after %d rows] %s", resultCounter, err)
if err != nil {
log.Errorf("DB - Archive of pokemon table (unverified timestamps) error [after %d rows] %s", resultCounter, err)
break
} else {
rows, _ := result.RowsAffected()
resultCounter += rows
if rows < databaseDeleteChunkSize {
elapsed := time.Since(start)
log.Infof("DB - Archive of pokemon table (unverified timestamps) took %s (%d rows)", elapsed, resultCounter)
break
} else {
rows, _ := result.RowsAffected()
resultCounter += rows
if rows < databaseDeleteChunkSize {
elapsed := time.Since(start)
log.Infof("DB - Archive of pokemon table (unverified timestamps) took %s (%d rows)", elapsed, resultCounter)
break
}
}
}
}
}()
}
}
}()

}

func StartStatsExpiry(db *sqlx.DB) {
Expand Down Expand Up @@ -270,33 +248,3 @@ func StartQuestExpiry(db *sqlx.DB) {
}
}()
}

func StartInMemoryCleardown(db *sqlx.DB) {
ticker := time.NewTicker(time.Minute)
go func() {
for {
<-ticker.C
start := time.Now()

var result sql.Result
var err error

unix := time.Now().Unix()

result, err = db.Exec(
fmt.Sprintf("DELETE FROM pokemon WHERE expire_timestamp < %d;",
unix-5*60))

elapsed := time.Since(start)

if err != nil {
log.Errorf("DB - Archive of pokemon table error %s", err)
return
}

rows, _ := result.RowsAffected()
log.Infof("DB - Cleardown of in-memory pokemon table took %s (%d rows)", elapsed, rows)

}
}()
}

0 comments on commit 198b5c5

Please sign in to comment.