Save seen locations to approximate marker frequency

Pal.Client/LocalState.cs

@@ -13,7 +13,7 @@ namespace Pal.Client
     internal class LocalState
     {
         private static readonly JsonSerializerOptions _jsonSerializerOptions = new JsonSerializerOptions { IncludeFields = true };
-        private static readonly int _currentVersion = 2;
+        private static readonly int _currentVersion = 3;
      
         public uint TerritoryType { get; set; }
         public ConcurrentBag<Marker> Markers { get; set; } = new();

Pal.Client/Marker.cs

@@ -1,6 +1,7 @@
 using ECommons.SplatoonAPI;
 using Palace;
 using System;
+using System.Collections.Generic;
 using System.Numerics;
 using System.Text.Json.Serialization;
 
@@ -10,18 +11,44 @@ namespace Pal.Client
     {
         public EType Type { get; set; } = EType.Unknown;
         public Vector3 Position { get; set; }
+
+        /// <summary>
+        /// Whether we have encountered the trap/coffer at this location in-game.
+        /// </summary>
         public bool Seen { get; set; } = false;
 
+        /// <summary>
+        /// Network id for the server you're currently connected to.
+        /// </summary>
         [JsonIgnore]
-        public bool RemoteSeen { get; set; } = false;
+        public Guid? NetworkId { get; set; }
+
+        /// <summary>
+        /// For markers that the server you're connected to doesn't know: Whether this was requested to be uploaded, to avoid duplicate requests.
+        /// </summary>
+        [JsonIgnore]
+        public bool UploadRequested { get; set; }
+
+        /// <summary>
+        /// Which account ids this marker was seen. This is a list merely to support different remote endpoints
+        /// (where each server would assign you a different id).
+        /// </summary>
+        public List<Guid> RemoteSeenOn { get; set; } = new List<Guid>();
+
+        /// <summary>
+        /// Whether this marker was requested to be seen, to avoid duplicate requests.
+        /// </summary>
+        [JsonIgnore]
+        public bool RemoteSeenRequested { get; set; } = false;
 
         [JsonIgnore]
         public Element? SplatoonElement { get; set; }
 
-        public Marker(EType type, Vector3 position)
+        public Marker(EType type, Vector3 position, Guid? networkId = null)
         {
             Type = type;
             Position = position;
+            NetworkId = networkId;
         }
 
         public override int GetHashCode()

Pal.Client/Pal.Client.csproj

@@ -3,7 +3,7 @@
 	<PropertyGroup>
 		<TargetFramework>net6.0-windows</TargetFramework>
 		<LangVersion>9.0</LangVersion>
-		<Version>1.11.0.0</Version>
+		<Version>1.12.0.0</Version>
 		<Nullable>enable</Nullable>
 	</PropertyGroup>
 

Pal.Client/Plugin.cs

@@ -13,6 +13,7 @@ using Grpc.Core;
 using ImGuiNET;
 using Lumina.Excel.GeneratedSheets;
 using Pal.Client.Windows;
+using Palace;
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
@@ -31,7 +32,7 @@ namespace Pal.Client
         private const string SPLATOON_TRAP_HOARD = "PalacePal.TrapHoard";
         private const string SPLATOON_REGULAR_COFFERS = "PalacePal.RegularCoffers";
 
-        private readonly ConcurrentQueue<(ushort territoryId, bool success, IList<Marker> markers)> _remoteDownloads = new();
+        private readonly ConcurrentQueue<Sync> _pendingSyncResponses = new();
         private readonly static Dictionary<Marker.EType, MarkerConfig> _markerConfig = new Dictionary<Marker.EType, MarkerConfig>
         {
             { Marker.EType.Trap, new MarkerConfig { Radius = 1.7f } },
@@ -256,9 +257,9 @@ namespace Pal.Client
                     Task.Run(async () => await DownloadMarkersForTerritory(LastTerritory));
                 }
 
-                if (_remoteDownloads.Count > 0)
+                if (_pendingSyncResponses.Count > 0)
                 {
-                    HandleRemoteDownloads();
+                    HandleSyncResponses();
                     recreateLayout = true;
                     saveMarkers = true;
                 }
@@ -281,6 +282,8 @@ namespace Pal.Client
             var config = Service.Configuration;
             var currentFloorMarkers = currentFloor.Markers;
 
+            bool updateSeenMarkers = false;
+            var accountId = Service.RemoteApi.AccountId;
             foreach (var visibleMarker in visibleMarkers)
             {
                 Marker? knownMarker = currentFloorMarkers.SingleOrDefault(x => x == visibleMarker);
@@ -291,6 +294,12 @@ namespace Pal.Client
                         knownMarker.Seen = true;
                         saveMarkers = true;
                     }
+
+                    // This requires you to have seen a trap/hoard marker once per floor to synchronize this for older local states,
+                    // markers discovered afterwards are automatically marked seen.
+                    if (accountId != null && knownMarker.NetworkId != null && !knownMarker.RemoteSeenRequested && !knownMarker.RemoteSeenOn.Contains(accountId.Value))
+                        updateSeenMarkers = true;
+                    
                     continue;
                 }
 
@@ -324,14 +333,24 @@ namespace Pal.Client
                 }
             }
 
+            if (updateSeenMarkers && accountId != null)
+            {
+                var markersToUpdate = currentFloorMarkers.Where(x => x.Seen && x.NetworkId != null && !x.RemoteSeenRequested && !x.RemoteSeenOn.Contains(accountId.Value)).ToList();
+                foreach (var marker in markersToUpdate)
+                    marker.RemoteSeenRequested = true;
+                Task.Run(async () => await SyncSeenMarkersForTerritory(LastTerritory, markersToUpdate));
+            }
+
             if (saveMarkers)
             {
                 currentFloor.Save();
 
                 if (TerritorySyncState == SyncState.Complete)
                 {
-                    var markersToUpload = currentFloorMarkers.Where(x => x.IsPermanent() && !x.RemoteSeen).ToList();
+                    var markersToUpload = currentFloorMarkers.Where(x => x.IsPermanent() && x.NetworkId == null && !x.UploadRequested).ToList();
-                    Task.Run(async () => await Service.RemoteApi.UploadMarker(LastTerritory, markersToUpload));
+                    foreach (var marker in markersToUpload)
+                        marker.UploadRequested = true;
+                    Task.Run(async () => await UploadMarkersForTerritory(LastTerritory, markersToUpload));
                 }
             }
 
@@ -442,7 +461,13 @@ namespace Pal.Client
             try
             {
                 var (success, downloadedMarkers) = await Service.RemoteApi.DownloadRemoteMarkers(territoryId);
-                _remoteDownloads.Enqueue((territoryId, success, downloadedMarkers));
+                _pendingSyncResponses.Enqueue(new Sync
+                {
+                    Type = SyncType.Download,
+                    TerritoryType = territoryId,
+                    Success = success,
+                    Markers = downloadedMarkers
+                });
             }
             catch (Exception e)
             {
@@ -450,6 +475,45 @@ namespace Pal.Client
             }
         }
 
+        private async Task UploadMarkersForTerritory(ushort territoryId, List<Marker> markersToUpload)
+        {
+            try
+            {
+                var (success, uploadedMarkers) = await Service.RemoteApi.UploadMarker(territoryId, markersToUpload);
+                _pendingSyncResponses.Enqueue(new Sync
+                {
+                    Type = SyncType.Upload,
+                    TerritoryType = territoryId,
+                    Success = success,
+                    Markers = uploadedMarkers
+                });
+            }
+            catch (Exception e)
+            {
+                DebugMessage = $"{DateTime.Now}\n{e}";
+            }
+        }
+
+        private async Task SyncSeenMarkersForTerritory(ushort territoryId, List<Marker> markersToUpdate)
+        {
+            try
+            {
+                var success = await Service.RemoteApi.MarkAsSeen(territoryId, markersToUpdate);
+                _pendingSyncResponses.Enqueue(new Sync
+                {
+                    Type = SyncType.MarkSeen,
+                    TerritoryType = territoryId,
+                    Success = success,
+                    Markers = markersToUpdate,
+                });
+            }
+            catch (Exception e)
+            {
+                DebugMessage = $"{DateTime.Now}\n{e}";
+            }
+        }
+
+
         private async Task FetchFloorStatistics()
         {
             if (Service.Configuration.Mode != Configuration.EMode.Online)
@@ -482,23 +546,46 @@ namespace Pal.Client
             }
         }
 
-        private void HandleRemoteDownloads()
+        private void HandleSyncResponses()
         {
-            while (_remoteDownloads.TryDequeue(out var download))
+            while (_pendingSyncResponses.TryDequeue(out Sync? sync) && sync != null)
             {
-                var (territoryId, success, downloadedMarkers) = download;
+                try
-                if (Service.Configuration.Mode == Configuration.EMode.Online && success && FloorMarkers.TryGetValue(territoryId, out var currentFloor) && downloadedMarkers.Count > 0)
                 {
-                    foreach (var downloadedMarker in downloadedMarkers)
+                    var territoryId = sync.TerritoryType;
+                    var remoteMarkers = sync.Markers;
+                    if (Service.Configuration.Mode == Configuration.EMode.Online && sync.Success && FloorMarkers.TryGetValue(territoryId, out var currentFloor) && remoteMarkers.Count > 0)
                     {
-                        Marker? seenMarker = currentFloor.Markers.SingleOrDefault(x => x == downloadedMarker);
+                        switch (sync.Type)
-                        if (seenMarker != null)
                         {
-                            seenMarker.RemoteSeen = true;
+                            case SyncType.Download:
+                            case SyncType.Upload:
+                                foreach (var remoteMarker in remoteMarkers)
+                                {
+                                    // Both uploads and downloads return the network id to be set, but only the downloaded marker is new as in to-be-saved.
+                                    Marker? localMarker = currentFloor.Markers.SingleOrDefault(x => x == remoteMarker);
+                                    if (localMarker != null)
+                                    {
+                                        localMarker.NetworkId = remoteMarker.NetworkId;
                                         continue;
                                     }
 
-                        currentFloor.Markers.Add(downloadedMarker);
+                                    if (sync.Type == SyncType.Download)
+                                        currentFloor.Markers.Add(remoteMarker);
+                                }
+                                break;
+
+                            case SyncType.MarkSeen:
+                                var accountId = Service.RemoteApi.AccountId;
+                                if (accountId == null)
+                                    break;
+                                foreach (var remoteMarker in remoteMarkers)
+                                {
+                                    Marker? localMarker = currentFloor.Markers.SingleOrDefault(x => x == remoteMarker);
+                                    if (localMarker != null)
+                                        localMarker.RemoteSeenOn.Add(accountId.Value);
+                                }
+                                break;
                         }
                     }
 
@@ -506,12 +593,22 @@ namespace Pal.Client
                     if (LastTerritory != territoryId)
                         continue;
 
-                if (success)
+                    if (sync.Type == SyncType.Download)
+                    {
+                        if (sync.Success)
                             TerritorySyncState = SyncState.Complete;
                         else
                             TerritorySyncState = SyncState.Failed;
                     }
                 }
+                catch (Exception e)
+                {
+                    DebugMessage = $"{DateTime.Now}\n{e}";
+                    if (sync.Type == SyncType.Download)
+                        TerritorySyncState = SyncState.Failed;
+                }
+            }
+        }
 
         private IList<Marker> GetRelevantGameObjects()
         {
@@ -588,14 +685,30 @@ namespace Pal.Client
             return Service.DataManager.GetExcelSheet<LogMessage>()?.GetRow(id)?.Text?.ToString() ?? "Unknown";
         }
 
+        internal class Sync
+        {
+            public SyncType Type { get; set; }
+            public ushort TerritoryType { get; set; }
+            public bool Success { get; set; }
+            public List<Marker> Markers { get; set; } = new();
+        }
+
         public enum SyncState
         {
             NotAttempted,
+            NotNeeded,
             Started,
             Complete,
             Failed,
         }
 
+        public enum SyncType
+        {
+            Upload,
+            Download,
+            MarkSeen,
+        }
+
         public enum PomanderState
         {
             Inactive,

Pal.Client/RemoteApi.cs

@@ -26,6 +26,18 @@ namespace Pal.Client
         private GrpcChannel? _channel;
         private LoginReply? _lastLoginReply;
 
+        public Guid? AccountId
+        {
+            get => Service.Configuration.AccountIds[remoteUrl];
+            set
+            {
+                if (value != null)
+                    Service.Configuration.AccountIds[remoteUrl] = value.Value;
+                else
+                    Service.Configuration.AccountIds.Remove(remoteUrl);
+            }
+        }
+
         private async Task<bool> Connect(CancellationToken cancellationToken, bool retry = true)
         {
             if (Service.Configuration.Mode != Configuration.EMode.Online)
@@ -47,30 +59,27 @@ namespace Pal.Client
             }
 
             var accountClient = new AccountService.AccountServiceClient(_channel);
-            Guid? accountId = Service.Configuration.AccountIds[remoteUrl];
+            if (AccountId == null)
-            if (accountId == null)
             {
                 var createAccountReply = await accountClient.CreateAccountAsync(new CreateAccountRequest(), headers: UnauthorizedHeaders(), deadline: DateTime.UtcNow.AddSeconds(10), cancellationToken: cancellationToken);
                 if (createAccountReply.Success)
                 {
-                    accountId = Guid.Parse(createAccountReply.AccountId);
+                    AccountId = Guid.Parse(createAccountReply.AccountId);
-                    Service.Configuration.AccountIds[remoteUrl] = accountId.Value;
                     Service.Configuration.Save();
                 }
             }
 
-            if (accountId == null)
+            if (AccountId == null)
                 return false;
 
             if (_lastLoginReply == null || string.IsNullOrEmpty(_lastLoginReply.AuthToken) || _lastLoginReply.ExpiresAt.ToDateTime().ToLocalTime() < DateTime.Now)
             {
-                _lastLoginReply = await accountClient.LoginAsync(new LoginRequest { AccountId = accountId.ToString() }, headers: UnauthorizedHeaders(), deadline: DateTime.UtcNow.AddSeconds(10), cancellationToken: cancellationToken);
+                _lastLoginReply = await accountClient.LoginAsync(new LoginRequest { AccountId = AccountId?.ToString() }, headers: UnauthorizedHeaders(), deadline: DateTime.UtcNow.AddSeconds(10), cancellationToken: cancellationToken);
                 if (!_lastLoginReply.Success)
                 {
                     if (_lastLoginReply.Error == LoginError.InvalidAccountId)
                     {
-                        accountId = null;
+                        AccountId = null;
-                        Service.Configuration.AccountIds.Remove(remoteUrl);
                         Service.Configuration.Save();
                         if (retry)
                             return await Connect(cancellationToken, retry: false);
@@ -100,16 +109,16 @@ namespace Pal.Client
 
             var palaceClient = new PalaceService.PalaceServiceClient(_channel);
             var downloadReply = await palaceClient.DownloadFloorsAsync(new DownloadFloorsRequest { TerritoryType = territoryId }, headers: AuthorizedHeaders(), cancellationToken: cancellationToken);
-            return (downloadReply.Success, downloadReply.Objects.Select(o => new Marker((Marker.EType)o.Type, new Vector3(o.X, o.Y, o.Z)) { RemoteSeen = true }).ToList());
+            return (downloadReply.Success, downloadReply.Objects.Select(o => CreateMarkerFromNetworkObject(o)).ToList());
         }
 
-        public async Task<bool> UploadMarker(ushort territoryType, IList<Marker> markers, CancellationToken cancellationToken = default)
+        public async Task<(bool, List<Marker>)> UploadMarker(ushort territoryType, IList<Marker> markers, CancellationToken cancellationToken = default)
         {
             if (markers.Count == 0)
-                return true;
+                return (true, new());
 
             if (!await Connect(cancellationToken))
-                return false;
+                return (false, new());
 
             var palaceClient = new PalaceService.PalaceServiceClient(_channel);
             var uploadRequest = new UploadFloorsRequest
@@ -124,9 +133,30 @@ namespace Pal.Client
                 Z = m.Position.Z
             }));
             var uploadReply = await palaceClient.UploadFloorsAsync(uploadRequest, headers: AuthorizedHeaders(), cancellationToken: cancellationToken);
-            return uploadReply.Success;
+            return (uploadReply.Success, uploadReply.Objects.Select(o => CreateMarkerFromNetworkObject(o)).ToList());
         }
 
+        public async Task<bool> MarkAsSeen(ushort territoryType, IList<Marker> markers, CancellationToken cancellationToken = default)
+        {
+            Service.Chat.Print($"Marking {markers.Count} as seen");
+            if (markers.Count == 0)
+                return true;
+
+            if (!await Connect(cancellationToken))
+                return false;
+
+            var palaceClient = new PalaceService.PalaceServiceClient(_channel);
+            var seenRequest = new MarkObjectsSeenRequest { TerritoryType = territoryType };
+            foreach (var marker in markers)
+                seenRequest.NetworkIds.Add(marker.NetworkId.ToString());
+
+            var seenReply = await palaceClient.MarkObjectsSeenAsync(seenRequest, headers: AuthorizedHeaders(), deadline: DateTime.UtcNow.AddSeconds(10), cancellationToken: cancellationToken);
+            return seenReply.Success;
+        }
+
+        private Marker CreateMarkerFromNetworkObject(PalaceObject obj) =>
+            new Marker((Marker.EType)obj.Type, new Vector3(obj.X, obj.Y, obj.Z), Guid.Parse(obj.NetworkId));
+        
         public async Task<(bool, List<FloorStatistics>)> FetchStatistics(CancellationToken cancellationToken = default)
         {
             if (!await Connect(cancellationToken))

Pal.Common/Protos/palace.proto

@@ -5,6 +5,7 @@ package palace;
 service PalaceService {
 	rpc DownloadFloors(DownloadFloorsRequest) returns (DownloadFloorsReply);
 	rpc UploadFloors(UploadFloorsRequest) returns (UploadFloorsReply);
+	rpc MarkObjectsSeen(MarkObjectsSeenRequest) returns (MarkObjectsSeenReply);
 	rpc FetchStatistics(StatisticsRequest) returns (StatisticsReply);
 }
 
@@ -24,6 +25,7 @@ message UploadFloorsRequest {
 
 message UploadFloorsReply {
 	bool success = 1;
+	repeated PalaceObject objects = 2;
 }
 
 message StatisticsRequest {
@@ -45,6 +47,18 @@ message PalaceObject {
 	float x = 2;
 	float y = 3;
 	float z = 4;
+
+	// Ignored for uploaded markers.
+	string networkId = 5;
+}
+
+message MarkObjectsSeenRequest {
+	uint32 territoryType = 1;
+	repeated string networkIds = 2;
+}
+
+message MarkObjectsSeenReply {
+	bool success = 1;
 }
 
 enum ObjectType {