Merge remote-tracking branch 'gagliardetto/multiepoch' into multiepoch

rpcpool · Aug 21, 2023 · aaabe14 · aaabe14
2 parents 383d1eb + c5d944a
commit aaabe14
Show file tree

Hide file tree

Showing 8 changed files with 1,623 additions and 744 deletions.
diff --git a/cmd-rpc-server-car-getBlock.go b/cmd-rpc-server-car-getBlock.go
@@ -470,8 +470,8 @@ func (ser *deprecatedRPCServer) handleGetBlock(ctx context.Context, conn *reques
 						})
 					return
 				}
-				parentEntryHash := solana.HashFromBytes(parentEntryNode.Hash)
-				blockResp.PreviousBlockhash = parentEntryHash.String()
+				parentEntryHash := solana.HashFromBytes(parentEntryNode.Hash).String()
+				blockResp.PreviousBlockhash = &parentEntryHash
 			}
 		}
 	}

diff --git a/cmd-rpc-server-car-getTransaction.go b/cmd-rpc-server-car-getTransaction.go
@@ -192,16 +192,56 @@ func adaptTransactionMetaToExpectedOutput(m map[string]any) map[string]any {
 			}
 			// remove loadedReadonlyAddresses and loadedWritableAddresses
 		}
-		if _, ok := meta["postTokenBalances"]; !ok {
+		if preTokenBalances, ok := meta["preTokenBalances"]; !ok {
+			meta["preTokenBalances"] = []any{}
+		} else {
+			// in preTokenBalances.[].uiTokenAmount.decimals if not present, set to 0
+			preTokenBalances, ok := preTokenBalances.([]any)
+			if ok {
+				for _, preTokenBalanceAny := range preTokenBalances {
+					preTokenBalance, ok := preTokenBalanceAny.(map[string]any)
+					if ok {
+						uiTokenAmountAny, ok := preTokenBalance["uiTokenAmount"]
+						if ok {
+							uiTokenAmount, ok := uiTokenAmountAny.(map[string]any)
+							if ok {
+								_, ok := uiTokenAmount["decimals"]
+								if !ok {
+									uiTokenAmount["decimals"] = 0
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+		if postTokenBalances, ok := meta["postTokenBalances"]; !ok {
 			meta["postTokenBalances"] = []any{}
+		} else {
+			// in postTokenBalances.[].uiTokenAmount.decimals if not present, set to 0
+			postTokenBalances, ok := postTokenBalances.([]any)
+			if ok {
+				for _, postTokenBalanceAny := range postTokenBalances {
+					postTokenBalance, ok := postTokenBalanceAny.(map[string]any)
+					if ok {
+						uiTokenAmountAny, ok := postTokenBalance["uiTokenAmount"]
+						if ok {
+							uiTokenAmount, ok := uiTokenAmountAny.(map[string]any)
+							if ok {
+								_, ok := uiTokenAmount["decimals"]
+								if !ok {
+									uiTokenAmount["decimals"] = 0
+								}
+							}
+						}
+					}
+				}
+			}
 		}
 		if _, ok := meta["returnDataNone"]; !ok {
 			// TODO: what is this?
 			meta["returnDataNone"] = nil
 		}
-		if _, ok := meta["preTokenBalances"]; !ok {
-			meta["preTokenBalances"] = []any{}
-		}
 		if _, ok := meta["rewards"]; !ok {
 			meta["rewards"] = []any{}
 		}
@@ -219,6 +259,19 @@ func adaptTransactionMetaToExpectedOutput(m map[string]any) map[string]any {
 				}
 			}
 		}
+		{
+			// TODO: is this correct?
+			// if doesn't have err, but has status and it is empty, then set status to Ok
+			if _, ok := meta["err"]; !ok || meta["err"] == nil {
+				if status, ok := meta["status"].(map[string]any); ok {
+					if len(status) == 0 {
+						meta["status"] = map[string]any{
+							"Ok": nil,
+						}
+					}
+				}
+			}
+		}
 	}
 	{
 		innerInstructionsAny, ok := meta["innerInstructions"]

diff --git a/cmd-rpc.go b/cmd-rpc.go
@@ -90,8 +90,10 @@ func newCmd_rpc() *cli.Command {
 			if err != nil {
 				return cli.Exit(err.Error(), 1)
 			}
-			klog.Infof("Found %d config files", len(configFiles))
-			spew.Dump(configFiles)
+			klog.Infof("Found %d config files:", len(configFiles))
+			for _, configFile := range configFiles {
+				fmt.Printf("  - %s\n", configFile)
+			}
 
 			// Load configs:
 			configs := make(ConfigSlice, 0)

diff --git a/cmd-x-index-all.go b/cmd-x-index-all.go
@@ -71,8 +71,8 @@ func newCmd_Index_all() *cli.Command {
 				if err != nil {
 					return err
 				}
-				spew.Dump(indexPaths)
-				klog.Info("Indexes created.")
+				klog.Info("Indexes created:")
+				veryPlainSdumpConfig.Dump(indexPaths)
 				if verify {
 					return verifyAllIndexes(context.Background(), carPath, indexPaths)
 				}
@@ -83,6 +83,16 @@ func newCmd_Index_all() *cli.Command {
 	}
 }
 
+var veryPlainSdumpConfig = spew.ConfigState{
+	Indent:                  "  ",
+	DisablePointerAddresses: true,
+	DisableCapacities:       true,
+	DisableMethods:          true,
+	DisablePointerMethods:   true,
+	ContinueOnMethod:        true,
+	SortKeys:                true,
+}
+
 func createAllIndexes(
 	ctx context.Context,
 	tmpDir string,
@@ -112,6 +122,10 @@ func createAllIndexes(
 	if len(rd.header.Roots) != 1 {
 		return nil, fmt.Errorf("car file must have exactly 1 root, but has %d", len(rd.header.Roots))
 	}
+	// print roots:
+	for _, root := range rd.header.Roots {
+		klog.Infof("- Root: %s", root)
+	}
 
 	klog.Infof("Getting car file size")
 	targetFileSize, err := getFileSize(carPath)
@@ -242,7 +256,7 @@ func createAllIndexes(
 		}
 		if numIndexedOffsets%10_000_000 == 0 {
 			timeFor10_000_000 := time.Since(lastCheckpoint)
-			howMany10_000_000 := (numTotalItems - numIndexedOffsets) / 10_000_000
+			howMany10_000_000 := ((numTotalItems - numIndexedOffsets) / 10_000_000) + 1
 			eta := timeFor10_000_000 * time.Duration(howMany10_000_000)
 
 			printToStderr(

diff --git a/multiepoch-getBlock.go b/multiepoch-getBlock.go
@@ -122,6 +122,7 @@ func (multi *MultiEpoch) handleGetBlock(ctx context.Context, conn *requestContex
 				if length > GiB {
 					length = GiB
 				}
+				klog.Infof("prefetching %d bytes from %d", length, parentOffset)
 				carSection, err := epochHandler.ReadAtFromCar(ctx, parentOffset, length)
 				if err != nil {
 					return err
@@ -132,7 +133,7 @@ func (multi *MultiEpoch) handleGetBlock(ctx context.Context, conn *requestContex
 
 				gotCid, data, err := util.ReadNode(br)
 				if err != nil {
-					return err
+					return fmt.Errorf("failed to read first node: %w", err)
 				}
 				if !parentIsInPreviousEpoch && !gotCid.Equals(parentCid) {
 					return fmt.Errorf("CID mismatch: expected %s, got %s", parentCid, gotCid)
@@ -294,12 +295,15 @@ func (multi *MultiEpoch) handleGetBlock(ctx context.Context, conn *requestContex
 							}
 						}
 					}
-					rewards = m["rewards"]
+					rewards = rewardsAsArray
 				} else {
 					klog.Errorf("did not find rewards field in rewards")
+					rewards = make([]any, 0)
 				}
 			}
 		}
+	} else {
+		rewards = make([]any, 0)
 	}
 	tim.time("get rewards")
 	{
@@ -385,8 +389,8 @@ func (multi *MultiEpoch) handleGetBlock(ctx context.Context, conn *requestContex
 						Message: "Internal error",
 					}, fmt.Errorf("failed to decode Entry: %v", err)
 				}
-				parentEntryHash := solana.HashFromBytes(parentEntryNode.Hash)
-				blockResp.PreviousBlockhash = parentEntryHash.String()
+				parentEntryHash := solana.HashFromBytes(parentEntryNode.Hash).String()
+				blockResp.PreviousBlockhash = &parentEntryHash
 			}
 		} else {
 			klog.Infof("parent slot is in a different epoch, not implemented yet")