Bbupdate rebase

.gitignore

@@ -8,6 +8,7 @@ debug*
 docker/blockbook
 build/pkg-defs
 build/blockbook
+build/blockchaincfg.json
 build/ldb
 build/sst_dump
 build/*.deb

Makefile

@@ -2,36 +2,35 @@ BIN_IMAGE = blockbook-build
 DEB_IMAGE = blockbook-build-deb
 PACKAGER = $(shell id -u):$(shell id -g)
 NO_CACHE = false
-UPDATE_VENDOR = 1
 ARGS ?=
 
 TARGETS=$(subst .json,, $(shell ls configs/coins))
 
 .PHONY: build build-debug test deb
 
 build: .bin-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -e UPDATE_VENDOR=$(UPDATE_VENDOR) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(BIN_IMAGE) make build ARGS="$(ARGS)"
+	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(BIN_IMAGE) make build ARGS="$(ARGS)"
 
 build-debug: .bin-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -e UPDATE_VENDOR=$(UPDATE_VENDOR) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(BIN_IMAGE) make build-debug ARGS="$(ARGS)"
+	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(BIN_IMAGE) make build-debug ARGS="$(ARGS)"
 
 test: .bin-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -e UPDATE_VENDOR=$(UPDATE_VENDOR) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test ARGS="$(ARGS)"
+	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test ARGS="$(ARGS)"
 
 test-integration: .bin-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -e UPDATE_VENDOR=$(UPDATE_VENDOR) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test-integration ARGS="$(ARGS)"
+	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test-integration ARGS="$(ARGS)"
 
 test-all: .bin-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -e UPDATE_VENDOR=$(UPDATE_VENDOR) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test-all ARGS="$(ARGS)"
+	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test-all ARGS="$(ARGS)"
 
 deb-backend-%: .deb-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -e UPDATE_VENDOR=$(UPDATE_VENDOR) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh backend $* $(ARGS)
+	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh backend $* $(ARGS)
 
 deb-blockbook-%: .deb-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -e UPDATE_VENDOR=$(UPDATE_VENDOR) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh blockbook $* $(ARGS)
+	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh blockbook $* $(ARGS)
 
 deb-%: .deb-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -e UPDATE_VENDOR=$(UPDATE_VENDOR) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh all $* $(ARGS)
+	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh all $* $(ARGS)
 
 deb-blockbook-all: clean-deb $(addprefix deb-blockbook-, $(TARGETS))
 

api/types.go

@@ -196,8 +196,7 @@ type Tx struct {
 	FeesSat          *Amount           `json:"fees,omitempty"`
 	Hex              string            `json:"hex,omitempty"`
 	Rbf              bool              `json:"rbf,omitempty"`
-	CoinSpecificData interface{}       `json:"-"`
-	CoinSpecificJSON json.RawMessage   `json:"-"`
+	CoinSpecificData json.RawMessage   `json:"coinSpecificData,omitempty"`
 	TokenTransfers   []TokenTransfer   `json:"tokenTransfers,omitempty"`
 	EthereumSpecific *EthereumSpecific `json:"ethereumSpecific,omitempty"`
 }
@@ -424,18 +423,19 @@ type BlockbookInfo struct {
 
 // BackendInfo is used to get information about blockchain
 type BackendInfo struct {
-	BackendError    string  `json:"error,omitempty"`
-	Chain           string  `json:"chain,omitempty"`
-	Blocks          int     `json:"blocks,omitempty"`
-	Headers         int     `json:"headers,omitempty"`
-	BestBlockHash   string  `json:"bestBlockHash,omitempty"`
-	Difficulty      string  `json:"difficulty,omitempty"`
-	SizeOnDisk      int64   `json:"sizeOnDisk,omitempty"`
-	Version         string  `json:"version,omitempty"`
-	Subversion      string  `json:"subversion,omitempty"`
-	ProtocolVersion string  `json:"protocolVersion,omitempty"`
-	Timeoffset      float64 `json:"timeOffset,omitempty"`
-	Warnings        string  `json:"warnings,omitempty"`
+	BackendError    string      `json:"error,omitempty"`
+	Chain           string      `json:"chain,omitempty"`
+	Blocks          int         `json:"blocks,omitempty"`
+	Headers         int         `json:"headers,omitempty"`
+	BestBlockHash   string      `json:"bestBlockHash,omitempty"`
+	Difficulty      string      `json:"difficulty,omitempty"`
+	SizeOnDisk      int64       `json:"sizeOnDisk,omitempty"`
+	Version         string      `json:"version,omitempty"`
+	Subversion      string      `json:"subversion,omitempty"`
+	ProtocolVersion string      `json:"protocolVersion,omitempty"`
+	Timeoffset      float64     `json:"timeOffset,omitempty"`
+	Warnings        string      `json:"warnings,omitempty"`
+	Consensus       interface{} `json:"consensus,omitempty"`
 }
 
 // SystemInfo contains information about the running blockbook and backend instance

api/worker.go

@@ -274,7 +274,8 @@ func (w *Worker) GetTransactionFromBchainTx(bchainTx *bchain.Tx, height int, spe
 	// for now do not return size, we would have to compute vsize of segwit transactions
 	// size:=len(bchainTx.Hex) / 2
 	var sj json.RawMessage
-	if specificJSON {
+	// return CoinSpecificData for all mempool transactions or if requested
+	if specificJSON || bchainTx.Confirmations == 0 {
 		sj, err = w.chain.GetTransactionSpecific(bchainTx)
 		if err != nil {
 			return nil, err
@@ -299,14 +300,140 @@ func (w *Worker) GetTransactionFromBchainTx(bchainTx *bchain.Tx, height int, spe
 		Rbf:              rbf,
 		Vin:              vins,
 		Vout:             vouts,
-		CoinSpecificData: bchainTx.CoinSpecificData,
-		CoinSpecificJSON: sj,
+		CoinSpecificData: sj,
 		TokenTransfers:   tokens,
 		EthereumSpecific: ethSpecific,
 	}
 	return r, nil
 }
 
+// GetTransactionFromMempoolTx converts bchain.MempoolTx to Tx, with limited amount of data
+// it is not doing any request to backend or to db
+func (w *Worker) GetTransactionFromMempoolTx(mempoolTx *bchain.MempoolTx) (*Tx, error) {
+	var err error
+	var valInSat, valOutSat, feesSat big.Int
+	var pValInSat *big.Int
+	var tokens []TokenTransfer
+	var ethSpecific *EthereumSpecific
+	vins := make([]Vin, len(mempoolTx.Vin))
+	rbf := false
+	for i := range mempoolTx.Vin {
+		bchainVin := &mempoolTx.Vin[i]
+		vin := &vins[i]
+		vin.Txid = bchainVin.Txid
+		vin.N = i
+		vin.Vout = bchainVin.Vout
+		vin.Sequence = int64(bchainVin.Sequence)
+		// detect explicit Replace-by-Fee transactions as defined by BIP125
+		if bchainVin.Sequence < 0xffffffff-1 {
+			rbf = true
+		}
+		vin.Hex = bchainVin.ScriptSig.Hex
+		vin.Coinbase = bchainVin.Coinbase
+		if w.chainType == bchain.ChainBitcoinType {
+			//  bchainVin.Txid=="" is coinbase transaction
+			if bchainVin.Txid != "" {
+				vin.ValueSat = (*Amount)(&bchainVin.ValueSat)
+				vin.AddrDesc = bchainVin.AddrDesc
+				vin.Addresses, vin.IsAddress, _ = w.chainParser.GetAddressesFromAddrDesc(vin.AddrDesc)
+				if vin.ValueSat != nil {
+					valInSat.Add(&valInSat, (*big.Int)(vin.ValueSat))
+				}
+			}
+		} else if w.chainType == bchain.ChainEthereumType {
+			if len(bchainVin.Addresses) > 0 {
+				vin.AddrDesc, err = w.chainParser.GetAddrDescFromAddress(bchainVin.Addresses[0])
+				if err != nil {
+					glog.Errorf("GetAddrDescFromAddress error %v, tx %v, bchainVin %v", err, mempoolTx.Txid, bchainVin)
+				}
+				vin.Addresses = bchainVin.Addresses
+				vin.IsAddress = true
+			}
+		}
+	}
+	vouts := make([]Vout, len(mempoolTx.Vout))
+	for i := range mempoolTx.Vout {
+		bchainVout := &mempoolTx.Vout[i]
+		vout := &vouts[i]
+		vout.N = i
+		vout.ValueSat = (*Amount)(&bchainVout.ValueSat)
+		valOutSat.Add(&valOutSat, &bchainVout.ValueSat)
+		vout.Hex = bchainVout.ScriptPubKey.Hex
+		vout.AddrDesc, vout.Addresses, vout.IsAddress, err = w.getAddressesFromVout(bchainVout)
+		if err != nil {
+			glog.V(2).Infof("getAddressesFromVout error %v, %v, output %v", err, mempoolTx.Txid, bchainVout.N)
+		}
+	}
+	if w.chainType == bchain.ChainBitcoinType {
+		// for coinbase transactions valIn is 0
+		feesSat.Sub(&valInSat, &valOutSat)
+		if feesSat.Sign() == -1 {
+			feesSat.SetUint64(0)
+		}
+		pValInSat = &valInSat
+	} else if w.chainType == bchain.ChainEthereumType {
+		if len(mempoolTx.Vout) > 0 {
+			valOutSat = mempoolTx.Vout[0].ValueSat
+		}
+		tokens = w.getTokensFromErc20(mempoolTx.Erc20)
+		ethTxData := eth.GetEthereumTxDataFromSpecificData(mempoolTx.CoinSpecificData)
+		ethSpecific = &EthereumSpecific{
+			GasLimit: ethTxData.GasLimit,
+			GasPrice: (*Amount)(ethTxData.GasPrice),
+			GasUsed:  ethTxData.GasUsed,
+			Nonce:    ethTxData.Nonce,
+			Status:   ethTxData.Status,
+			Data:     ethTxData.Data,
+		}
+	}
+	r := &Tx{
+		Blocktime:        mempoolTx.Blocktime,
+		FeesSat:          (*Amount)(&feesSat),
+		Locktime:         mempoolTx.LockTime,
+		Txid:             mempoolTx.Txid,
+		ValueInSat:       (*Amount)(pValInSat),
+		ValueOutSat:      (*Amount)(&valOutSat),
+		Version:          mempoolTx.Version,
+		Hex:              mempoolTx.Hex,
+		Rbf:              rbf,
+		Vin:              vins,
+		Vout:             vouts,
+		TokenTransfers:   tokens,
+		EthereumSpecific: ethSpecific,
+	}
+	return r, nil
+}
+
+func (w *Worker) getTokensFromErc20(erc20 []bchain.Erc20Transfer) []TokenTransfer {
+	tokens := make([]TokenTransfer, len(erc20))
+	for i := range erc20 {
+		e := &erc20[i]
+		cd, err := w.chainParser.GetAddrDescFromAddress(e.Contract)
+		if err != nil {
+			glog.Errorf("GetAddrDescFromAddress error %v, contract %v", err, e.Contract)
+			continue
+		}
+		erc20c, err := w.chain.EthereumTypeGetErc20ContractInfo(cd)
+		if err != nil {
+			glog.Errorf("GetErc20ContractInfo error %v, contract %v", err, e.Contract)
+		}
+		if erc20c == nil {
+			erc20c = &bchain.Erc20Contract{Name: e.Contract}
+		}
+		tokens[i] = TokenTransfer{
+			Type:     ERC20TokenType,
+			Token:    e.Contract,
+			From:     e.From,
+			To:       e.To,
+			Decimals: erc20c.Decimals,
+			Value:    (*Amount)(&e.Tokens),
+			Name:     erc20c.Name,
+			Symbol:   erc20c.Symbol,
+		}
+	}
+	return tokens
+}
+
 // GetTransactionFromMempoolTx converts bchain.MempoolTx to Tx, with limited amount of data
 // it is not doing any request to backend or to db
 func (w *Worker) GetTransactionFromMempoolTx(mempoolTx *bchain.MempoolTx) (*Tx, error) {
@@ -758,7 +885,7 @@ func (w *Worker) txFromTxid(txid string, bestheight uint32, option AccountDetail
 		if ta == nil {
 			glog.Warning("DB inconsistency:  tx ", txid, ": not found in txAddresses")
 			// as fallback, get tx from backend
-			tx, err = w.GetTransaction(txid, false, true)
+			tx, err = w.GetTransaction(txid, false, false)
 			if err != nil {
 				return nil, errors.Annotatef(err, "GetTransaction %v", txid)
 			}
@@ -777,7 +904,7 @@ func (w *Worker) txFromTxid(txid string, bestheight uint32, option AccountDetail
 			tx = w.txFromTxAddress(txid, ta, blockInfo, bestheight)
 		}
 	} else {
-		tx, err = w.GetTransaction(txid, false, true)
+		tx, err = w.GetTransaction(txid, false, false)
 		if err != nil {
 			return nil, errors.Annotatef(err, "GetTransaction %v", txid)
 		}
@@ -866,7 +993,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option Acco
 			return nil, errors.Annotatef(err, "getAddressTxids %v true", addrDesc)
 		}
 		for _, txid := range txm {
-			tx, err := w.GetTransaction(txid, false, false)
+			tx, err := w.GetTransaction(txid, false, true)
 			// mempool transaction may fail
 			if err != nil || tx == nil {
 				glog.Warning("GetTransaction in mempool: ", err)
@@ -1777,6 +1904,7 @@ func (w *Worker) GetSystemInfo(internal bool) (*SystemInfo, error) {
 		Timeoffset:      ci.Timeoffset,
 		Version:         ci.Version,
 		Warnings:        ci.Warnings,
+		Consensus:       ci.Consensus,
 	}
 	glog.Info("GetSystemInfo finished in ", time.Since(start))
 	return &SystemInfo{blockbookInfo, backendInfo}, nil

api/xpub.go

@@ -416,7 +416,7 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Acc
 					// the same tx can have multiple addresses from the same xpub, get it from backend it only once
 					tx, foundTx := txmMap[txid.txid]
 					if !foundTx {
-						tx, err = w.GetTransaction(txid.txid, false, false)
+						tx, err = w.GetTransaction(txid.txid, false, true)
 						// mempool transaction may fail
 						if err != nil || tx == nil {
 							glog.Warning("GetTransaction in mempool: ", err)

bchain/coins/eth/ethparser.go

@@ -311,8 +311,14 @@ func (p *EthereumParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) (
 		if pt.Receipt.GasUsed, err = hexDecodeBig(r.Receipt.GasUsed); err != nil {
 			return nil, errors.Annotatef(err, "GasUsed %v", r.Receipt.GasUsed)
 		}
-		if pt.Receipt.Status, err = hexDecodeBig(r.Receipt.Status); err != nil {
-			return nil, errors.Annotatef(err, "Status %v", r.Receipt.Status)
+		if r.Receipt.Status != "" {
+			if pt.Receipt.Status, err = hexDecodeBig(r.Receipt.Status); err != nil {
+				return nil, errors.Annotatef(err, "Status %v", r.Receipt.Status)
+			}
+		} else {
+			// unknown status, use 'U' as status bytes
+			// there is a potential for conflict with value 0x55 but this is not used by any chain at this moment
+			pt.Receipt.Status = []byte{'U'}
 		}
 		ptLogs := make([]*ProtoCompleteTransaction_ReceiptType_LogType, len(r.Receipt.Logs))
 		for i, l := range r.Receipt.Logs {
@@ -379,9 +385,14 @@ func (p *EthereumParser) UnpackTx(buf []byte) (*bchain.Tx, uint32, error) {
 				Topics:  topics,
 			}
 		}
+		status := ""
+		// handle a special value []byte{'U'} as unknown state
+		if len(pt.Receipt.Status) != 1 || pt.Receipt.Status[0] != 'U' {
+			status = hexEncodeBig(pt.Receipt.Status)
+		}
 		rr = &rpcReceipt{
 			GasUsed: hexEncodeBig(pt.Receipt.GasUsed),
-			Status:  hexEncodeBig(pt.Receipt.Status),
+			Status:  status,
 			Logs:    logs,
 		}
 	}