Merge pull request #674 from net-zero-project/main

Update O&G data loader

Author: sichen1234

.env.SAMPLE

@@ -45,9 +45,9 @@ LEDGER_FIRST_BLOCK="0"
 #LEDGER_ETH_JSON_RPC_URL="https://goerli.infura.io/v3/<your-infura-project-id>"
 #LEDGER_ETH_WS_URL="wss://goerli.infura.io/ws/v3/<your-infura-project-id>"
 #LEDGER_ETH_NETWORK="goerli"
-#LEDGER_EMISSION_TOKEN_CONTRACT_ADDRESS="0xE8854ec567b3762046d8f773547f373e6A1A620A"
+#LEDGER_EMISSION_TOKEN_CONTRACT_ADDRESS="0x118938E52100df8FfE8793f8ec231f56d85e8E72"
 #LEDGER_FIRST_BLOCK="7385471"
-#LEDGER_CARBON_TRACKER_ADDRESS="0xDb80143bCf04B53Ed586b7f806C25ff275b5Cc5b"
+#LEDGER_CARBON_TRACKER_ADDRESS="0x039EF5c92CC8249D2cb3785B68bc4F79dC8a7625"
 
 # it should be 2048 for Avalanche testnet and could be 5000 for bsc testnet.  2048 seems to work for Goerli
 LEDGER_EVENTS_BLOCK_INTERVAL="2048"

app/api-server/controller/synchronizer.ts

@@ -305,7 +305,7 @@ const getNumOfProductTokens = async (opts: OPTS_TYPE): Promise<number> => {
         return result;
     } catch (err) {
         console.error(err)
-        throw new Error('Error in _numOfProducts: ' + err)
+        throw new Error('Error in getNumOfProductTokens: ' + err)
     }
 }
 

app/api-server/server.ts

@@ -26,7 +26,7 @@ const assertEnv = (key: string): string => {
   return process.env[key] || '';
 }
 const contract_address = assertEnv('LEDGER_EMISSION_TOKEN_CONTRACT_ADDRESS')
-const tracker_address = process.env['LEDGER_CARBON_TRACKER_ADDRESS']
+const tracker_address = assertEnv('LEDGER_CARBON_TRACKER_ADDRESS')
 const network_name = assertEnv('LEDGER_ETH_NETWORK')
 const network_rpc_url = assertEnv('LEDGER_ETH_JSON_RPC_URL')
 const network_ws_url = process.env['LEDGER_ETH_WS_URL']

app/frontend/contracts/src/addresses.ts

@@ -21,6 +21,24 @@ const networksAndAddresses = {
     }
   },
 
+  hedera_testnet: {
+    network: "Hedera Testnet",
+    tokenNetwork: {
+      address: "0x00000000000000000000000000000000003be83f", 
+    },
+    carbonTracker: {
+      address: "0x00000000000000000000000000000000003c0064",
+    },
+    dao: {
+      governor: {
+        address: "",
+      },
+      daoToken: {
+        address: "",
+      }
+    }
+  },
+
   goerli: {
     network: "Goerli Testnet",
     tokenNetwork: {
@@ -34,10 +52,10 @@ const networksAndAddresses = {
     },
     dao: {
       governor: {
-        address: "0x9F4590684d1DD950dF65Dc71D39f782bc3af317A",
+        address: "0x6875735ee7864673B211eC2ec2a534179F863DAD",
       },
       daoToken: {
-        address: "0x67E5b82809f28aA321818b90bF5c603D954f87C8",
+        address: "0x08AA8Ae1E7A087b591a5a1234fBE20e592c5c417",
       }
     }
   },
@@ -152,6 +170,6 @@ const networksAndAddresses = {
 }
 
 // change this to the network you want
-const addresses = networksAndAddresses.hardhat;
+const addresses = networksAndAddresses.hedera_testnet;
 
 export default addresses;

app/frontend/react-app/src/components/tracker-info-modal.tsx

@@ -107,12 +107,11 @@ const TrackerInfoModal:FC<TrackerInfoModalProps> = ({provider,show,tracker,onHid
             : null)}
             <tr>
               <td>From date</td>
-              {/*TO-DO infer fromDate/thruDate from Net Emission tokens*/}
-              <td><DisplayDate date={tracker.fromDate}/></td>
+              <td><DisplayDate date={Math.min.apply(Math,tracker?.tokens?.map(e=>{return e?.token?.thruDate!}) as number[])}/></td>
             </tr>
             <tr>
               <td>Thru date</td>
-              <td><DisplayDate date={tracker.thruDate}/></td>
+              <td><DisplayDate date={Math.min.apply(Math,tracker?.tokens?.map(e=>{return e?.token?.thruDate!}) as number[])}/></td>
             </tr>
             <tr>
               <td>Description</td>

app/methane/client/src/components/static-data.tsx

@@ -1,6 +1,7 @@
 import type { Field } from "@blockchain-carbon-accounting/react-app/src/components/static-data"
 export type Asset = {
   name?: string
+  operator?: string
   latitude: number
   longitude: number
   division_type?: string
@@ -77,6 +78,12 @@ export const ASSET_FIELDS: Field[] = [
     type: 'string',
     ops: ['eq']
 },
+{
+    alias: 'Operator',
+    name: 'operator',
+    type: 'string',
+    ops: ['like']
+},
 ]
 
 export const PRODUCT_FIELDS: Field[] = [

app/methane/client/src/pages/assets.tsx

@@ -182,7 +182,7 @@ const RegisteredOperator: ForwardRefRenderFunction<AssetsHandle, AssetProps> = (
         <div className="mt-4">
           <h2 style={{display: 'inline'}}>
             Operator: {operator?.name}&nbsp;
-            {assetCount} Assets&nbsp;
+            {operator?.assetsCount} ({assetCount}) Assets &nbsp;
           </h2>
           &nbsp;
           <Button className="mb-3" onClick={switchQueryBuilder} variant={(showQueryBuilder) ? 'dark' : 'outline-dark'}><BsFunnel /></Button>
@@ -197,6 +197,7 @@ const RegisteredOperator: ForwardRefRenderFunction<AssetsHandle, AssetProps> = (
             <thead>
               <tr>
                 <th>Name</th>
+                <th>Operator</th>
                 <th>State</th>
                 <th>Status</th>
                 <th></th>
@@ -206,7 +207,8 @@ const RegisteredOperator: ForwardRefRenderFunction<AssetsHandle, AssetProps> = (
               {!!selectedAssets &&
                 selectedAssets.map((asset,index) => (
                   <tr key={[asset?.name,index].join('_')} onClick={() => handleOpenOperatorInfoModal(asset)} onMouseOver={pointerHover}>
-                    <td> {asset.name}</td>
+                    <td>{asset.name}</td>
+                    <td>{asset?.operator}</td>
                     <td>{asset?.division_name}</td>
                     <td>{asset?.status}</td>
                     <td>

data/loadOGdata.sh

@@ -1,4 +1,6 @@
-npm run dataLoader load_og_assets "./oil_and_gas/files/Oil_and_Natural_Gas_Wells.geojson" -- --format US_asset_data --source "https://hifld-geoplatform.opendata.arcgis.com/datasets/geoplatform::oil-and-natural-gas-wells/explore"
+#npm run dataLoader load_og_assets "./oil_and_gas/files/Oil_and_Natural_Gas_Wells.geojson" -- --format US_asset_data --source "https://hifld-geoplatform.opendata.arcgis.com/datasets/geoplatform::oil-and-natural-gas-wells/explore"
+#npm run dataLoader set_asset_operators
+
 npm run dataLoader load_product_data "./oil_and_gas/files/2022-og-benchmarking-report-data.xlsx" "Company_basin" -- --format Benchmark --source "https://www.sustainability.com/globalassets/sustainability.com/thinking/pdfs/2022/2022-og-benchmarking-report-data.xlsx" --skip_rows 7
 
 npm run dataLoader load_product_data "./oil_and_gas/files/VIIRS_Global_flaring_d.7_slope_0.029353_2021_web.xlsx" "flare upstream" -- --format VIIRS --year 2021 --source "https://eogdata.mines.edu/global_flare_data/VIIRS_Global_flaring_d.7_slope_0.029353_2021_web.xlsx"

data/oil_and_gas/README.md

@@ -9,7 +9,24 @@ Or you can download the data
 sh download.sh
 ```
 
-Once you have the data, load them into postgres db (blockchain-carbon-accounting) by running the following from `/data`:
+### Oil & gas asset and operator data.
+
+The oil & gas asset data includes >1.5 million assets.
+To avoid reloading the data into postres `download.sh` returns 3 binary files that can be used to restore the following postrgres tables (after initializing the DB with `npm run pg:init`)
+
+1. oil_and_gas_asset
+2. operator
+3. asset_operator
+
+Otherwise the tables can be loaded using the following commands from `\data`.
+
+``` npm run dataLoader load_og_assets "./oil_and_gas/files/Oil_and_Natural_Gas_Wells.geojson" -- --format US_asset_data --source "https://hifld-geoplatform.opendata.arcgis.com/datasets/geoplatform::oil-and-natural-gas-wells/explore"```
+
+```npm run dataLoader set_asset_operators```
+
+### Oil & gas product & emissions data
+
+Load the product data into postgres db by running the following from `/data` directorty:
 ```
 sh loadOGdata.sh
 ```

data/oil_and_gas/download.sh

@@ -9,6 +9,11 @@ curl -L -c cookies.txt 'https://docs.google.com/uc?export=download&id='$fileId \
 curl -L -b cookies.txt -o $fileName \
      'https://docs.google.com/uc?export=download&id='$fileId'&confirm='$(<confirm.txt)
 rm -f confirm.txt cookies.txt
+echo "Downloading postgres table backup..."
+curl \
+-L -o 'asset_operator' 'https://docs.google.com/uc?export=download&id=1ME0XIz065OEMReqfvsVOl7oW5ye2BmUl' \
+-L -o 'operator' 'https://docs.google.com/uc?export=download&id=1MHKrkMvq9FZCw_RYsiIrXi1rRmfLmSD7' \
+-L -o 'oil_and_gas_asset' 'https://docs.google.com/uc?export=download&id=1VPYAKasx3SJ9l3wuW99CMMFHEGU-4Dfr'
 
 echo "Downloading oil and gas product data from google drive..."
 echo "VIIRS flaring data..."