fix: restore standalone ipld instance (#289)

This PR partially  reverts
#287
because `/api/v0/dag` in go-ipfs do not implement latest IPLD features
yet, and ipfs-http-client delegates path traversal and resolution to
go-ipfs, so codecs in JS are never used.

By reverting to standalone js-ipld we decouple IPLD explorer from what
is available in go-ipfs – we simply fetch raw block and do all decoding
the old way, in JS.

We will revisit this in the future, after IPLD Prime work lands in
go-ipfs, but for now this is the only way to fix IPLD Explorer to work
against go-ipfs backend.

Author: lidel

package-lock.json

@@ -18,28 +18,33 @@
     "start": "cross-env NODE_ENV=production babel src -d dist --copy-files --ignore '**/*.test.js' --watch",
     "storybook": "start-storybook -p 9009 --static-dir public",
     "storybook:build": "build-storybook -c .storybook --static-dir public --output-dir build",
-    "test": "react-scripts test"
+    "test": "react-scripts test --env=jsdom"
   },
   "dependencies": {
-    "@babel/cli": "^7.13.10",
+    "@babel/cli": "^7.13.14",
     "@loadable/component": "^5.14.1",
     "@tableflip/react-inspector": "^2.3.0",
     "cids": "^1.1.6",
     "cytoscape": "^3.18.1",
     "cytoscape-dagre": "^2.3.2",
     "filesize": "^6.1.0",
     "ipfs-unixfs": "^4.0.1",
+    "ipld": "0.29.0",
+    "ipld-dag-cbor": "0.18.0",
+    "ipld-ethereum": "6.0.0",
+    "ipld-git": "0.6.4",
+    "ipld-raw": "7.0.0",
     "milliseconds": "^1.0.3",
     "multibase": "^4.0.4",
     "multicodec": "^3.0.1",
     "multihashes": "^4.0.2",
     "react-joyride": "^2.3.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.13.10",
+    "@babel/core": "^7.13.15",
     "@babel/plugin-proposal-class-properties": "^7.13.0",
-    "@babel/preset-env": "^7.13.12",
-    "@babel/preset-react": "^7.12.13",
+    "@babel/preset-env": "^7.13.15",
+    "@babel/preset-react": "^7.13.13",
     "@storybook/addon-a11y": "^5.3.21",
     "@storybook/addon-actions": "^5.3.21",
     "@storybook/addon-knobs": "^5.3.21",
@@ -64,7 +69,7 @@
     "i18next-localstorage-backend": "3.1.2",
     "intl-messageformat": "^9.6.4",
     "ipfs-css": "^1.3.0",
-    "ipld-dag-pb": "^0.22.1",
+    "ipld-dag-pb": "0.22.2",
     "react": "^16.14.0",
     "react-dom": "^16.14.0",
     "react-helmet": "^5.2.1",

package.json

@@ -5,6 +5,10 @@ import parseIpldPath from '../lib/parse-ipld-path'
 
 // Find all the nodes and path boundaries traversed along a given path
 const makeBundle = () => {
+  // Lazy load ipld because it is a large dependency
+  let IpldResolver = null
+  let ipldFormats = null
+
   const bundle = createAsyncResourceBundle({
     name: 'explore',
     actionBaseType: 'EXPLORE',
@@ -16,15 +20,22 @@ const makeBundle = () => {
       if (!pathParts) return null
       const { cidOrFqdn, rest } = pathParts
       try {
+        if (!IpldResolver) {
+          const { ipld, formats } = await getIpld()
+
+          IpldResolver = ipld
+          ipldFormats = formats
+        }
+        const ipld = makeIpld(IpldResolver, ipldFormats, getIpfs)
+        // TODO: handle ipns, which would give us a fqdn in the cid position.
         const cid = new Cid(cidOrFqdn)
-        const ipfs = await getIpfs()
         const {
           targetNode,
           canonicalPath,
           localPath,
           nodes,
           pathBoundaries
-        } = await resolveIpldPath(ipfs, cid, rest)
+        } = await resolveIpldPath(ipld, cid, rest)
 
         return {
           path,
@@ -101,4 +112,31 @@ function ensureLeadingSlash (str) {
   return `/${str}`
 }
 
+function makeIpld (IpldResolver, ipldFormats, getIpfs) {
+  return new IpldResolver({
+    blockService: getIpfs().block,
+    formats: ipldFormats
+  })
+}
+
+async function getIpld () {
+  const ipldDeps = await Promise.all([
+    import(/* webpackChunkName: "ipld" */ 'ipld'),
+    import(/* webpackChunkName: "ipld" */ 'ipld-dag-cbor'),
+    import(/* webpackChunkName: "ipld" */ 'ipld-dag-pb'),
+    import(/* webpackChunkName: "ipld" */ 'ipld-git'),
+    import(/* webpackChunkName: "ipld" */ 'ipld-raw'),
+    import(/* webpackChunkName: "ipld" */ 'ipld-ethereum')
+  ])
+
+  // CommonJs exports object is .default when imported ESM style
+  const [ipld, ...formats] = ipldDeps.map(mod => mod.default)
+
+  // ipldEthereum is an Object, each key points to a ipld format impl
+  const ipldEthereum = formats.pop()
+  formats.push(...Object.values(ipldEthereum))
+
+  return { ipld, formats }
+}
+
 export default makeBundle

src/bundles/explore.js

@@ -36,13 +36,13 @@ const StartExploringPage = ({ t, embed, runTour = false, joyrideCallback }) => (
             <ExploreSuggestion name='Project Apollo Archives' cid='QmSnuWmxptJZdLJpKRarxBMS2Ju2oANVrgbr2xWbie9b2D' type='dag-pb' />
           </li>
           <li>
-            <ExploreSuggestion name='IGIS git repo' cid='baf4bcfg4ep767tjp5lxyanx5urpjjgx5q2volvy' type='git-raw' />
+            <ExploreSuggestion name='IGIS Git Repo' cid='baf4bcfg4ep767tjp5lxyanx5urpjjgx5q2volvy' type='git-raw' />
           </li>
           <li>
             <ExploreSuggestion name='An Ethereum Block' cid='bagiacgzah24drzou2jlkixpblbgbg6nxfrasoklzttzoht5hixhxz3rlncyq' type='eth-block' />
           </li>
           <li>
-            <ExploreSuggestion name='XKCD' cid='QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm' type='dag-pb' />
+            <ExploreSuggestion name='XKCD Archives' cid='QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm' type='dag-pb' />
           </li>
         </ul>
       </div>
@@ -63,10 +63,10 @@ const StartExploringPage = ({ t, embed, runTour = false, joyrideCallback }) => (
 
 /* TODO: add dag-cbor and raw block examples
           <li>
-            <ExploreSuggestion name='DAG-CBOR Block' cid='bafkreibm6jg3ux5qumhcn2b3flc3tyu6dmlb4xa7u5bf44yegnrjhc4yeq' type='dag-cbor' />
+            <ExploreSuggestion name='DAG-CBOR Block' cid='bafyreicnokmhmrnlp2wjhyk2haep4tqxiptwfrp2rrs7rzq7uk766chqvq' type='dag-cbor' />
           </li>
           <li>
-            <ExploreSuggestion name='Plain text as raw bytes' cid='bafkreibm6jg3ux5qumhcn2b3flc3tyu6dmlb4xa7u5bf44yegnrjhc4yeq' type='raw' />
+            <ExploreSuggestion name='Raw Block for "hello"' cid='bafkreibm6jg3ux5qumhcn2b3flc3tyu6dmlb4xa7u5bf44yegnrjhc4yeq' type='raw' />
           </li>
 */
 

src/components/StartExploringPage.js

@@ -17,8 +17,8 @@ const objectInspectorTheme = {
 
 // TODO: Use https://github.com/multiformats/multicodec/blob/master/table.csv to get full name.
 const nodeStyles = {
-  'dag-cbor': { shortName: 'CBOR', name: 'CBOR', color: '#28CA9F' },
-  'dag-pb': { shortName: 'PB', name: 'Protobuf', color: '#244e66' },
+  'dag-cbor': { shortName: 'CBOR', name: 'dag-cbor', color: '#28CA9F' },
+  'dag-pb': { shortName: 'PB', name: 'dag-pb', color: '#244e66' },
   'git-raw': { shortName: 'GIT', name: 'Git', color: '#378085' },
   'raw': { shortName: 'RAW', name: 'Raw Block', color: '#f14e32' }, // eslint-disable-line quote-props
   'eth-block': { shortName: 'ETH', name: 'Ethereum Block', color: '#383838' },
@@ -71,7 +71,7 @@ const ObjectInfo = ({ t, tReady, className, type, cid, localPath, size, data, li
           {nameForNode(type)}
         </span>
         {format === 'unixfs' ? (
-          <a className='dn di-ns link charcoal ml2' href='https://docs.ipfs.io/concepts/file-systems/#unix-file-system-unixfs'>UnixFS</a>
+          <a className='dn di-ns link charcoal ml2' href='https://docs.ipfs.io/concepts/file-systems/#unix-file-system-unixfs' target='_external'>UnixFS</a>
         ) : null}
         {format === 'unixfs' && data.type && ['directory', 'file'].some(x => x === data.type) ? (
           <a className='link avenir ml2 pa2 fw5 f6 blue' href={`${gatewayUrl}/ipfs/${cid}`} target='_external'>

src/components/object-info/ObjectInfo.js

@@ -17,5 +17,5 @@ export function getCodecOrNull (value) {
 
 export function toCidStrOrNull (value) {
   const cid = toCidOrNull(value)
-  return cid ? cid.toBaseEncodedString() : null
+  return cid ? cid.toString() : null
 }

src/lib/cid.js

@@ -35,8 +35,8 @@ import Cid from 'cids'
  * @param {Object[]} pathBoundaries accumulated path boundary info
  * @returns {{targetNode: Object, canonicalPath: String, localPath: String, nodes: Object[], pathBoundaries: Object[]}} resolved path info
  */
-export default async function resolveIpldPath (ipfs, sourceCid, path, nodes = [], pathBoundaries = []) {
-  const { value, remainderPath } = await ipldGetNodeAndRemainder(ipfs, sourceCid, path)
+export default async function resolveIpldPath (ipld, sourceCid, path, nodes = [], pathBoundaries = []) {
+  const { value, remainderPath } = await ipldGetNodeAndRemainder(ipld, sourceCid, path)
   const sourceCidStr = sourceCid.toString()
 
   const node = normaliseDagNode(value, sourceCidStr)
@@ -47,17 +47,28 @@ export default async function resolveIpldPath (ipfs, sourceCid, path, nodes = []
   if (link) {
     pathBoundaries.push(link)
     // Go again, using the link.target as the sourceCid, and the remainderPath as the path.
-    return resolveIpldPath(ipfs, new Cid(link.target), remainderPath, nodes, pathBoundaries)
+    return resolveIpldPath(ipld, new Cid(link.target), remainderPath, nodes, pathBoundaries)
   }
   // we made it to the containing node. Hand back the info
   const canonicalPath = path ? `${sourceCidStr}${path}` : sourceCidStr
   const targetNode = node
   return { targetNode, canonicalPath, localPath: path, nodes, pathBoundaries }
 }
 
-export async function ipldGetNodeAndRemainder (ipfs, sourceCid, path) {
-  const { value, remainderPath } = await ipfs.dag.get(sourceCid, path, { localResolve: true })
-  return { value, remainderPath }
+export async function ipldGetNodeAndRemainder (ipld, sourceCid, path) {
+  // TODO: find out why ipfs.dag.get with localResolve never resolves.
+  // const {value, remainderPath} = await getIpfs().dag.get(sourceCid, path, {localResolve: true})
+
+  // TODO: use ipfs.dag.get when it gets ipld super powers
+  // SEE: https://github.com/ipfs/js-ipfs-api/pull/755
+  // const {value} = await getIpfs().dag.get(sourceCid)
+
+  // TODO: handle indexing into dag-pb links without using Links prefix as per go-ipfs dag.get does.
+  // Current js-ipld-dag-pb resolver will throw with a path not available error if Links prefix is missing.
+  return {
+    value: await ipld.get(sourceCid),
+    remainderPath: (await ipld.resolve(sourceCid, path || '/').first()).remainderPath
+  }
 }
 
 /**

src/lib/resolve-ipld-path.js

@@ -4,33 +4,37 @@ import { DAGNode } from 'ipld-dag-pb'
 import resolveIpldPath, { findLinkPath } from './resolve-ipld-path'
 
 it('resolves all nodes traversed along a path', async () => {
-  const ipfsMock = {
-    dag: { get: jest.fn() }
+  const ipldMock = {
+    get: jest.fn(),
+    resolve: jest.fn()
   }
   const cid = 'zdpuAs8sJjcmsPUfB1bUViftCZ8usnvs2cXrPH6MDyT4zrvSs'
   const path = '/a/b/a'
   const linkCid = 'zdpuAyzU5ahAKr5YV24J5TqrDX8PhzHLMkxx69oVzkBDWHnjq'
   const dagGetRes1 = {
-    value: {
-      a: {
-        b: new CID(linkCid)
-      }
-    },
-    remainderPath: '/a'
+    a: {
+      b: new CID(linkCid)
+    }
   }
   const dagGetRes2 = {
-    value: {
-      a: 'hello world'
-    },
-    remainderPath: '/a'
+    first: () => Promise.resolve({ remainderPath: '/a' })
+  }
+  const dagGetRes3 = {
+    a: 'hello world'
+  }
+  const dagGetRes4 = {
+    first: () => Promise.resolve({ remainderPath: '/a' })
   }
 
-  ipfsMock.dag.get.mockReturnValueOnce(Promise.resolve(dagGetRes1))
-  ipfsMock.dag.get.mockReturnValueOnce(Promise.resolve(dagGetRes2))
+  ipldMock.get.mockReturnValueOnce(Promise.resolve(dagGetRes1))
+  ipldMock.resolve.mockReturnValueOnce(dagGetRes2)
+  ipldMock.get.mockReturnValueOnce(Promise.resolve(dagGetRes3))
+  ipldMock.resolve.mockReturnValueOnce(dagGetRes4)
 
-  const res = await resolveIpldPath(ipfsMock, new CID(cid), path)
+  const res = await resolveIpldPath(ipldMock, new CID(cid), path)
 
-  expect(ipfsMock.dag.get.mock.calls.length).toBe(2)
+  expect(ipldMock.get.mock.calls.length).toBe(2)
+  expect(ipldMock.resolve.mock.calls.length).toBe(2)
   expect(res.canonicalPath).toBe(`${linkCid}/a`)
   expect(res.nodes.length).toBe(2)
   expect(res.nodes[0].type).toBe('dag-cbor')
@@ -45,20 +49,19 @@ it('resolves all nodes traversed along a path', async () => {
   })
 })
 
-it('resolves thru dag-cbor to dag-pb', async () => {
-  const ipfsMock = {
-    dag: {
-      get: jest.fn()
-    }
+it('resolves thru dag-cbor to dag-pb to dag-pb', async () => {
+  const ipldMock = {
+    get: jest.fn(),
+    resolve: jest.fn()
   }
 
   const cid = 'zdpuAs8sJjcmsPUfB1bUViftCZ8usnvs2cXrPH6MDyT4zrvSs'
   const path = '/a/b/pb1'
 
-  const dagNode3 = await createDagPbNode(new Uint8Array(Buffer.from('the second pb node')), [])
+  const dagNode3 = await createDagPbNode('the second pb node', [])
   const dagNode3CID = 'QmRLacjo71FTzKFELa7Yf5YqMwdftKNDNFq7EiE13uohar'
 
-  const dagNode2 = await createDagPbNode(new Uint8Array(Buffer.from('the first pb node')), [{
+  const dagNode2 = await createDagPbNode('the first pb node', [{
     name: 'pb1',
     cid: dagNode3CID,
     size: 101
@@ -71,28 +74,35 @@ it('resolves thru dag-cbor to dag-pb', async () => {
     }
   }
 
-  const dagGetRes1 = {
-    value: dagNode1,
-    remainderPath: 'pb1'
-  }
+  const dagGetRes1 = dagNode1
 
   const dagGetRes2 = {
-    value: dagNode2,
-    remainderPath: ''
+    first: () => Promise.resolve({ remainderPath: 'pb1' })
   }
 
-  const dagGetRes3 = {
-    value: dagNode3,
-    remainderPath: ''
+  const dagGetRes3 = dagNode2
+
+  const dagGetRes4 = {
+    first: () => Promise.resolve({ remainderPath: '' })
   }
 
-  ipfsMock.dag.get.mockReturnValueOnce(Promise.resolve(dagGetRes1))
-  ipfsMock.dag.get.mockReturnValueOnce(Promise.resolve(dagGetRes2))
-  ipfsMock.dag.get.mockReturnValueOnce(Promise.resolve(dagGetRes3))
+  const dagGetRes5 = dagNode3
+
+  const dagGetRes6 = {
+    first: () => Promise.resolve({ remainderPath: '' })
+  }
 
-  const res = await resolveIpldPath(ipfsMock, new CID(cid), path)
+  ipldMock.get.mockReturnValueOnce(Promise.resolve(dagGetRes1))
+  ipldMock.resolve.mockReturnValueOnce(dagGetRes2)
+  ipldMock.get.mockReturnValueOnce(Promise.resolve(dagGetRes3))
+  ipldMock.resolve.mockReturnValueOnce(dagGetRes4)
+  ipldMock.get.mockReturnValueOnce(Promise.resolve(dagGetRes5))
+  ipldMock.resolve.mockReturnValueOnce(dagGetRes6)
 
-  expect(ipfsMock.dag.get.mock.calls.length).toBe(3)
+  const res = await resolveIpldPath(ipldMock, new CID(cid), path)
+
+  expect(ipldMock.get.mock.calls.length).toBe(3)
+  expect(ipldMock.resolve.mock.calls.length).toBe(3)
   expect(res.targetNode.cid).toEqual(dagNode3CID)
   expect(res.canonicalPath).toBe(dagNode3CID)
   expect(res.nodes.length).toBe(3)
@@ -121,13 +131,10 @@ it('resolves thru dag-cbor to dag-pb', async () => {
 })
 
 function createDagPbNode (data, links) {
-  const node = new DAGNode(data)
-
-  for (const link of links) {
-    node.addLink(link)
+  if (typeof data === 'string') {
+    data = new Uint8Array(Buffer.from(data))
   }
-
-  return node
+  return new DAGNode(data, links)
 }
 
 it('finds the linkPath from a fullPath and a remainderPath', () => {