Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 | 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 130x 130x 130x 130x 130x 130x 130x 130x 130x 130x 130x 130x 2x 2x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 2x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 42x 2x 60x 60x 76x 76x 76x 76x 76x 76x 60x 60x 60x 60x 60x 56x 56x 56x 56x 56x 60x 5x 5x 5x 5x 5x 5x 5x 55x 60x 7x 7x 7x 6x 7x 2x 2x 5x 5x 5x 5x 5x 5x 48x 60x 60x 60x 35x 35x 13x 13x 13x 13x 17x 16x 60x 8x 8x 8x 2x 2x 2x 2x 6x 6x 8x 8x 8x 7x 60x 5x 60x 2x 9x 9x 5x 5x 5x 1049x 5x 5x 9x 2x 27x 27x 27x 4x 4x 4x 4x 4x 4x 27x 2x 19x 19x 2x 12x 12x 12x 2x 3x 3x 2x 16786x 16786x 16786x 16785x 16786x 2x 16x 16x 16x 16x 16x 16x 16x 12x 12x 12x 12x 12x 12x 12x 12x 12x 12x 12x 16x 16x 16x 16x 16x 2x 17835x 17835x 17871x 17871x 17871x 17871x 17728x 17728x 17728x 17728x 143x 143x 17871x 143x 143x 17871x 143x 17871x 143x 143x 143x 143x 17320x 270x 270x 270x 270x 142x 17835x 2x 143x 131x 131x 143x 384x 384x 143x 46x 46x 143x 2x 143x 143x 143x 143x 143x 143x 143x 143x 2x 155x 1x 1x 1x 155x 155x 155x 155x 2x 155x 155x 155x 2x 155x 155x 155x 155x 155x 155x 155x 155x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 155x 155x 155x 155x 5x 155x 2x 270x 270x 270x 270x 280x 10x 10x 10x 10x 10x 270x 2x 17728x 17728x 17728x 17728x 17728x 2x 17870x 17870x 17870x 17870x 17870x 17870x 17870x 17870x 17870x 17870x 17870x 17870x 61x 61x 61x 61x 61x 7x 7x 7x 7x 7x 7x 7x 7x 7x 7x 1x 1x 7x 57x 57x 57x 7x 7x 72x 72x 72x 58x 72x 72x 155x 155x 155x 155x 155x 155x 155x 155x 155x 155x 155x 155x 12x 12x 6x 12x 6x 12x 12x 5x 5x 5x 5x 5x | /**
* A remote file as a {@link ByteSource}, read over HTTP range requests
* (spec §11, M5).
*
* The shape is Emscripten's `FS.createLazyFile` — an FS node whose contents are
* fetched on demand — but the built-in is not reused, for three reasons that
* are all requirements rather than preferences:
*
* 1. **It probes with a synchronous `HEAD` of its own.** We must probe
* asynchronously first anyway, to tell "this host does not serve ranges"
* apart from "the browser refused the request" (spec §10); doing it twice
* wastes a request out of a budget of twenty.
* 2. **Its cache is unbounded.** Every chunk ever touched is retained, so a
* long session over a large file ends up holding the file. h5wasm's own
* `lazyFileLRU` demo exists to fix exactly this, and this is that fix.
* 3. **It counts nothing.** The §12 budgets are assertions on request and byte
* counts, so the source has to be the thing that counts them.
*
* Reads must be synchronous because Emscripten's stream `read` is synchronous,
* so this only works where synchronous `XMLHttpRequest` is allowed: **inside a
* worker**. That is where all I/O lives anyway. The command line has no such
* request, and hands a synchronous fetch of its own instead
* ({@link RemoteRangeOptions.syncFetch}, `cli/syncFetch.ts`).
*/
import type { ByteSource, SourceStats } from '../types'
import { ViewerError } from '../errors'
import { getH5 } from '../h5/bootstrap'
import { mountLazyFile, unmountLazyFile } from '../h5/lazyFile'
import type { S3Credential } from '../s3/sigv4'
import { accessHeaders } from './access'
import { isSignedUrl } from './rewrite'
/** 1 MiB. Large enough that HDF5's metadata walk lands repeatedly in one chunk. */
const DEFAULT_CHUNK_SIZE = 1 << 20
/** 64 MiB of chunks retained. Well above one MAG's matrix, far below a file. */
const DEFAULT_CACHE_BYTES = 64 << 20
/**
* How close a miss must land to a recent fetch to count as the same region
* being read — see {@link RemoteRangeSource.aheadFor}.
*/
const NEAR_BYTES = 4 << 20
/** The most one miss fetches beyond what it needs. */
const MAX_AHEAD_BYTES = 16 << 20
/** Recent fetches remembered, so several streams read side by side are each seen. */
const RECENT_FETCHES = 8
export interface RemoteRangeOptions {
/** Bytes per cached chunk. */
chunkSize?: number
/**
* Called after every request, with the running totals.
*
* For the one read that cannot report itself: listing a single-cell
* container's cells is a thousand synchronous requests inside one h5wasm
* call, and the thread is not free to say anything until it returns. What
* it *can* do is say so from inside each request, which is enough for a
* bar to count them.
*/
onRead?(requests: number, bytes: number): void
/** Upper bound on retained chunk bytes; least-recently-used are dropped. */
maxCacheBytes?: number
/**
* Fetch ahead of a region being read chunk after chunk (default on) — see
* {@link RemoteRangeSource.aheadFor}. Off for a source opened to read in small
* pieces, whose reader chose bandwidth over round trips (§12).
*/
readAhead?: boolean
/** Injected for tests. Defaults to the global `fetch`. */
fetchImpl?: typeof fetch
/**
* The synchronous request every chunk is fetched with. A synchronous
* `XMLHttpRequest` where there is none given — a browser worker's — and the
* command line's own where there is not one of those (`cli/syncFetch.ts`).
*/
syncFetch?: SyncRangeFetch
/**
* The credential to sign every request with, for a private bucket.
*
* Omitted for a public object, and that is not merely an optimisation:
* an `Authorization` header turns each request into a preflight plus a real
* request, and a bucket policy scoped to one prefix would *reject* a signed
* read of a public one. Unsigned is the correct request for public data, not
* a weaker version of the signed one.
*
* This is the reason `s3/sha256.ts` exists. The signature has to be computable
* inside {@link RemoteRangeSource.fetchRange}, which is synchronous because
* Emscripten's stream `read` op is, so there is no `await` available and
* `crypto.subtle` — whose every method returns a promise — cannot be reached
* from there.
*/
credential?: S3Credential
/**
* Static headers for every request this source makes (SPECS §24.7).
*
* An API key or a bearer token, decided on the main thread where the storage
* is. Unlike the credential above they are not recomputed per request — there
* is nothing in them that goes stale within a session — so they are simply
* held and merged in front of the signature by `sources/access.ts`.
*/
headers?: Record<string, string>
}
/** What one synchronous range request answered. */
export interface SyncRangeAnswer {
status: number
body: Uint8Array
/** The `Content-Range` header, or `null` where the response had none. */
contentRange: string | null
}
/**
* A synchronous `GET` of `url` with `headers` — `Range` among them — or a throw
* where no response arrived.
*/
export type SyncRangeFetch = (url: string, headers: Record<string, string>) => SyncRangeAnswer
/** The browser's synchronous fetch: an `XMLHttpRequest`, legal only inside a worker. */
function xhrRange(url: string, headers: Record<string, string>): SyncRangeAnswer {
const xhr = new XMLHttpRequest()
xhr.open('GET', url, false)
for (const [name, value] of Object.entries(headers)) xhr.setRequestHeader(name, value)
// Allowed on a synchronous request only inside a worker; the guard in
// mount() is what makes this reachable.
xhr.responseType = 'arraybuffer'
xhr.send(null)
return {
status: xhr.status,
body: new Uint8Array((xhr.response as ArrayBuffer | null) ?? new ArrayBuffer(0)),
contentRange: xhr.getResponseHeader('content-range'),
}
}
export class RemoteRangeSource implements ByteSource {
readonly id: string
readonly size: number
/** Always: every seek this source answers is a range request. */
readonly isRemote = true
readonly url: string
private readonly chunkSize: number
private readonly onRead: ((requests: number, bytes: number) => void) | undefined
private readonly maxCacheBytes: number
/** Insertion-ordered, so the first key is the least recently used. */
private readonly cache = new Map<number, Uint8Array>()
private cachedBytes = 0
private readonly readAhead: boolean
/**
* The last few fetches, newest last: their chunk range, and how many misses
* in a row had landed near the region each was part of.
*/
private readonly recent: Array<{ from: number; to: number; streak: number }> = []
private requests: number
private bytesRead: number
private mountPath: string | null = null
private readonly credential: S3Credential | undefined
private readonly headers: Record<string, string> | undefined
/** What {@link readExact} fetches through — the probe's own, so a test's stub serves both. */
private readonly fetchImpl: typeof fetch
/** What every chunk is fetched through, or `null` for the worker's own `XMLHttpRequest`. */
private readonly syncFetch: SyncRangeFetch | null
private constructor(
url: string,
size: number,
options: RemoteRangeOptions,
probeCost: SourceStats,
) {
this.url = url
this.id = url
this.size = size
this.credential = options.credential
this.headers = options.headers
// Wrapped rather than held: a browser's `fetch` called as a method of this
// object throws "Illegal invocation".
this.fetchImpl =
options.fetchImpl ?? ((input: RequestInfo | URL, init?: RequestInit) => globalThis.fetch(input, init))
this.syncFetch = options.syncFetch ?? null
this.chunkSize = Math.max(1, Math.trunc(options.chunkSize ?? DEFAULT_CHUNK_SIZE))
this.readAhead = options.readAhead ?? true
this.onRead = options.onRead
const requested = Math.trunc(options.maxCacheBytes ?? DEFAULT_CACHE_BYTES)
this.maxCacheBytes = Math.max(this.chunkSize, requested)
this.requests = probeCost.requests
this.bytesRead = probeCost.bytes
}
/**
* Probe the URL, then return a source ready to mount.
*
* The probe is what produces the two distinct refusals spec §10 demands. A
* cross-origin failure surfaces in the browser as an opaque `TypeError` on
* `fetch` with no status and no headers — indistinguishable, on its face,
* from the host being down. It is told apart by *shape*: a second request in
* `no-cors` mode does not require CORS headers, so if that one succeeds while
* the plain request failed, the host is reachable and the response was
* blocked by the browser's CORS check.
*
* ## The `HEAD` is skipped for a URL that carries its own signature
*
* A presigned S3 URL and a signed GCS one are signed for **one method**, so
* the probe below is refused where the `GET` behind it would have been served
* — which is what §11 recorded as "presigned URLs do not work" and what SPECS
* §24.6 now unwinds. There is nothing to add for it: the `bytes=0-0` probe is
* already the road taken by every host that says nothing about ranges (4DN's
* bucket is one), and it yields both answers the `HEAD` would have. So a
* signed URL takes that road from the start, and a link that expires opens and
* later stops opening with the `403` that says so.
*/
static async open(url: string, options: RemoteRangeOptions = {}): Promise<RemoteRangeSource> {
const doFetch = options.fetchImpl ?? globalThis.fetch
// Signed per request, not once for the source: `x-amz-date` is part of the
// signature and S3 refuses a stamp more than fifteen minutes old, so a
// header bag computed at open time would start failing partway through a
// long session — on a file that had been reading fine, which is the worst
// shape a failure can have. The static headers beside it do not go stale and
// are merged in front of it by `sources/access.ts`.
const sign = (method: string): Record<string, string> =>
accessHeaders({
url,
method,
...(options.headers === undefined ? {} : { headers: options.headers }),
...(options.credential === undefined ? {} : { credential: options.credential }),
})
if (typeof doFetch !== 'function') {
throw new ViewerError('remote-cors', 'this runtime has no fetch(); remote files need one', {
url,
})
}
const host = hostOf(url)
const cost: SourceStats = { requests: 0, bytes: 0 }
let head: Response | null = null
if (!isSignedUrl(url)) {
try {
cost.requests += 1
head = await doFetch(url, { method: 'HEAD', headers: sign('HEAD') })
} catch (cause) {
await failCrossOrigin(url, host, cause, doFetch, cost)
}
}
if (head && !head.ok) {
// An HTTP error status is neither of spec §10's two remote refusals: the
// request was allowed through and answered, it just did not name a
// readable object. `bad-uri` is the honest code for that.
//
// The advice splits on whether a key was used, because the two situations
// share a status code and share nothing else. Told the public story, a
// reader with a private bucket goes looking for a permissions problem on
// an object that is *meant* to be private; told the private story, a
// reader with no key configured goes hunting for a key they never had.
throw new ViewerError(
'bad-uri',
options.credential === undefined
? `${host} answered ${head.status} ${head.statusText} to a HEAD of ${url}; ` +
'the viewer needs a publicly readable object, or a bucket connected with a key ' +
'for it (connect a bucket…, on the landing page or in the Files column).'
: `${host} answered ${head.status} ${head.statusText} to a signed HEAD of ${url}. ` +
'The key reached the bucket and was rejected: check that the region on the ' +
'bucket entry matches the bucket, that the key is allowed `s3:GetObject` on ' +
'this prefix, and — for a temporary credential — that the session token has ' +
'not expired.',
{ url, host, status: head.status, signed: options.credential !== undefined },
)
}
// Before anything is believed about sizes or ranges: a host that re-encodes
// the body makes both meaningless. `Range` addresses the octets of the
// *encoded* representation, so `bytes=1048576-2097151` is a slice out of the
// middle of a gzip stream — not a valid one — and `Content-Length` is the
// compressed size, several MB short of the file. The browser then hands back
// however many bytes that slice inflated to, which for the very first read
// is a correct prefix of the file: HDF5 opens, walks a few objects, and
// fails deep inside itself with `error - name not defined!`.
//
// This is not hypothetical or exotic. GitHub Pages compresses
// `application/octet-stream`, which is what it serves every `.metacool`,
// `.mcool` and `.cool` as, and no client can opt out — `Accept-Encoding` is
// a forbidden header name, so neither `fetch` nor `XMLHttpRequest` may set
// it. The caller's recourse is to stop using ranges (see
// `worker/sources.ts`), which is why this carries its own code.
const encoding = reEncoding(head)
if (encoding !== null) {
// A host that re-encodes an *unranged* answer may still serve a range
// raw. Google Drive is one: a browser says it accepts gzip, and Drive
// gzips the HEAD — `Content-Encoding: gzip`, chunked, no length — while
// every ranged request, HEAD or GET, is answered 206 with the identity
// bytes. So the range is asked for by HEAD before anything is concluded:
// a 206 that is not re-encoded and states its length is a host the reads
// below can use, and the length is the file's, since `bytes=0-` is all
// of it. A host that re-encodes that too (GitHub Pages) is refused as
// before, one request later.
const ranged = await rangedHead(url, doFetch, sign, cost)
const rangedLength =
totalFromContentRange(ranged?.headers.get('content-range') ?? null) ??
numberHeader(ranged, 'content-length')
if (ranged !== null && ranged.status === 206 && reEncoding(ranged) === null && rangedLength !== null) {
return new RemoteRangeSource(url, rangedLength, options, cost)
}
throw new ViewerError(
'remote-compressed',
// The diagnosis only; see the sibling refusal above. The advice under it
// already names object storage and GitHub Pages, word for word.
`${host} is serving ${url} with “Content-Encoding: ${encoding}”, so byte ranges ` +
'address the compressed stream rather than the file.',
{ url, host, encoding, encodedLength: numberHeader(head, 'content-length') },
)
}
// A `HEAD` that states no bytes has stated nothing: jupyter-server-proxy —
// the way into a kernel's files behind JupyterHub — answers every `HEAD`
// with `Content-Length: 0` whatever the file is, and believing it mounted a
// map as a file of no bytes ("not an HDF5 file"). A file that really is
// empty is nothing to open either, and the probe below says which.
const statedLength = numberHeader(head, 'content-length')
const declaredLength = statedLength === 0 ? null : statedLength
const acceptsRanges = (head?.headers.get('accept-ranges') ?? '').toLowerCase().includes('bytes')
if (acceptsRanges && declaredLength !== null) {
return new RemoteRangeSource(url, declaredLength, options, cost)
}
// Either the host says nothing about ranges, or CORS hid the headers from
// us. One byte settles both questions and yields the size along the way.
let probe: Response
try {
cost.requests += 1
probe = await doFetch(url, { method: 'GET', headers: { ...sign('GET'), Range: 'bytes=0-0' } })
} catch (cause) {
await failCrossOrigin(url, host, cause, doFetch, cost)
throw new Error('unreachable')
}
cost.bytes += numberHeader(probe, 'content-length') ?? 1
if (probe.status !== 206) {
throw new ViewerError(
'remote-no-ranges',
// **The diagnosis, and only the diagnosis.** What to do about it is the
// advice `ui/ErrorView.ts` prints directly underneath, and this said the
// same two things first, in different words: the reader was told that the
// viewer reads in slices and where to host the file, twice, an inch
// apart. What the advice cannot know is which host, which URL and what
// came back instead of a 206, so that is what stays here.
//
// An error status is not a host that ignores `Range`: it is the
// address's own answer — no file, or not for this page — and the page
// heads it as a refusal (`ErrorView`, `REFUSED_GUIDANCE`). It is where
// a presigned link arrives once it has expired, since it skips the
// `HEAD` above, so a signed address says so.
probe.status >= 400
? `${host} answered ${probe.status} ${probe.statusText} to a request for the first ` +
`byte of ${url}` +
(isSignedUrl(url)
? ': this is a presigned link, and a host refuses one once it has expired.'
: '.')
: `${host} does not serve HTTP range requests: a “Range: bytes=0-0” request for ` +
`${url} returned ${probe.status}, not 206 Partial Content.`,
{ url, host, status: probe.status },
)
}
const total = totalFromContentRange(probe.headers.get('content-range')) ?? declaredLength
if (total === null) {
throw new ViewerError(
'remote-cors',
`${host} serves ranges for ${url} but does not expose its size to this page: ` +
'neither “Content-Length” nor “Content-Range” was readable. Add both to the ' +
'bucket’s CORS “Access-Control-Expose-Headers”.',
{ url, host },
)
}
return new RemoteRangeSource(url, total, options, cost)
}
/**
* Create the lazy FS node and return its path.
*
* A file whose every read is {@link copyInto}, mirroring what
* `FS.createLazyFile` builds — in MEMFS in a worker, and beside the real
* filesystem under the command line's NODERAWFS (`h5/lazyFile.ts`).
*/
async mount(): Promise<string> {
if (this.mountPath !== null) return this.mountPath
if (this.syncFetch === null && typeof XMLHttpRequest === 'undefined') {
throw new ViewerError(
'remote-cors',
'remote files are read with synchronous XMLHttpRequest, which exists only in a ' +
'browser worker; this runtime has none.',
{ url: this.url },
)
}
const h5 = await getH5()
const leaf = `remote-${Math.abs(hash(this.url)).toString(36)}`
this.mountPath = mountLazyFile(h5, leaf, this.size, (buffer, offset, position, length) =>
this.copyInto(buffer, offset, position, length),
)
return this.mountPath
}
close(): void {
this.cache.clear()
this.cachedBytes = 0
if (this.mountPath === null) return
const path = this.mountPath
this.mountPath = null
void getH5().then(
(h5) => unmountLazyFile(h5, path),
() => undefined,
)
}
/** Requests issued and bytes received, including the open-time probe. */
stats(): SourceStats {
return { requests: this.requests, bytes: this.bytesRead }
}
/** Zero the counters, so a budget test can measure one view change. */
resetStats(): void {
this.requests = 0
this.bytesRead = 0
}
/** Chunks currently retained — a cache-behaviour assertion for tests. */
cachedChunkCount(): number {
return this.cache.size
}
/**
* Read `[position, position + length)` of the remote file.
*
* This is the byte accessor {@link mount}'s stream op is built on, exposed
* because the chunk cache is the part with the interesting behaviour —
* coalescing and eviction — and it should be assertable without standing up
* an Emscripten FS.
*/
readBytes(position: number, length: number): Uint8Array {
const n = Math.max(0, Math.min(this.size - position, length))
const out = new Uint8Array(n)
if (n > 0) this.copyInto(out, 0, position, n)
return out
}
/**
* Read `[position, position + length)` in one request for exactly that range.
*
* The cache is consulted and never filled: a range whose chunks are all held
* is copied out of them for nothing, and any other is fetched as asked for
* and handed over whole. Filling the cache from it would need the chunks
* either side, which is the megabyte this exists not to fetch — see
* `ByteSource.readExact`.
*
* Through `fetch`, not the synchronous request every other read makes, so the
* worker is free while it is in flight and a walk can have several at once.
* The request is otherwise the same one, refused the same ways.
*/
async readExact(position: number, length: number): Promise<Uint8Array> {
const n = Math.max(0, Math.min(this.size - position, length))
if (n === 0) return new Uint8Array(0)
const first = Math.floor(position / this.chunkSize)
const last = Math.floor((position + n - 1) / this.chunkSize)
let held = true
for (let i = first; i <= last && held; i++) held = this.cache.has(i)
if (held) return this.readBytes(position, n)
const from = position
const to = position + n - 1
const doFetch = this.fetchImpl
const response = await doFetch(this.url, {
method: 'GET',
headers: { ...this.authorisation(), Range: `bytes=${from}-${to}` },
})
this.requests += 1
this.refuseStatus(response.status, from, to)
const body = new Uint8Array(await response.arrayBuffer())
this.bytesRead += body.byteLength
this.onRead?.(this.requests, this.bytesRead)
const fetched = this.windowOf(body, from, to, response.headers.get('content-range'))
if (fetched.byteLength < n) {
throw new ViewerError('remote-no-ranges', 'a range request returned a short response', {
wanted: n,
available: fetched.byteLength,
})
}
// Copied only when the window is a slice of a bigger body — a host that sent
// more than was asked for — so that body is not kept alive by a few bytes.
return fetched.byteLength === n && fetched.buffer.byteLength === n ? fetched : fetched.slice(0, n)
}
/**
* Copy `length` bytes starting at `position` into `buffer` at `offset`,
* fetching whatever is missing.
*
* Contiguous missing chunks are fetched in **one** range request. That
* coalescing is what keeps a bounded pixel slice inside the §12 budget of
* five requests: without it, a 4 MB slice at 1 MiB chunks would be four.
*
* Bytes are copied straight out of the response and only *then* offered to
* the cache. Doing it the other way round — fill the cache, read it back —
* looks tidier and is wrong: a read spanning more chunks than the cache
* holds evicts its own earlier chunks before it gets to them.
*/
private copyInto(buffer: Uint8Array, offset: number, position: number, length: number): void {
let done = 0
while (done < length) {
const at = position + done
const index = Math.floor(at / this.chunkSize)
const cached = this.cache.get(index)
if (cached) {
this.touch(index)
done += blit(buffer, offset + done, cached, at - index * this.chunkSize, length - done)
continue
}
const last = Math.floor((position + length - 1) / this.chunkSize)
let end = index
while (end + 1 <= last && !this.cache.has(end + 1)) end += 1
// Beyond what this read needs, where the reads before it say more of the
// same region is coming — but never over a chunk already held.
const { ahead, streak } = this.aheadFor(index, end)
const beyond = Math.min(end + ahead, Math.ceil(this.size / this.chunkSize) - 1)
while (end < beyond && !this.cache.has(end + 1)) end += 1
this.recent.push({ from: index, to: end, streak })
if (this.recent.length > RECENT_FETCHES) this.recent.shift()
const from = index * this.chunkSize
const to = Math.min((end + 1) * this.chunkSize, this.size) - 1
const fetched = this.fetchRange(from, to)
done += blit(buffer, offset + done, fetched, at - from, length - done)
for (let i = index; i <= end; i++) {
const start = i * this.chunkSize - from
if (start >= fetched.length) break
this.store(i, fetched.subarray(start, Math.min(start + this.chunkSize, fetched.length)))
}
}
}
/**
* How many chunks to fetch past the ones a miss needs.
*
* HDF5 reads a cooler's pixels a dataset chunk at a time — `bin1_id`,
* `bin2_id` and `count` in turn, each a megabyte or so — and each read that
* misses the cache is a round trip, one after another on the worker. Reading a
* whole chromosome of a 5 GB `.mcool` was sixty-three of them, a megabyte
* apiece, hopping between three streams through one 60 MB region of the file:
* the latency, paid sixty-three times, was the wait.
*
* So a miss that lands within {@link NEAR_BYTES} of one of the last few
* fetches is taken as that region still being read, and from the third such
* miss in a row the fetch runs ahead, doubling each time up to
* {@link MAX_AHEAD_BYTES}. The count is kept per region, so a read of the
* weights or the index between two pixel chunks does not start the pixels'
* count again. A miss far from anything recent is a new question — an index,
* a header, another sequence — and fetches what it asks for and nothing more,
* so the scattered reads of opening a file are what they were.
*/
private aheadFor(index: number, end: number): { ahead: number; streak: number } {
if (!this.readAhead) return { ahead: 0, streak: 0 }
const near = Math.max(1, Math.floor(NEAR_BYTES / this.chunkSize))
let streak = 0
for (const fetch of this.recent) {
if (index >= fetch.from - near && index <= fetch.to + near) streak = Math.max(streak, fetch.streak + 1)
}
if (streak < 2) return { ahead: 0, streak }
const most = Math.max(0, Math.floor(MAX_AHEAD_BYTES / this.chunkSize) - (end - index + 1))
return { ahead: Math.min(most, 2 ** (streak - 2)), streak }
}
private fetchRange(from: number, to: number): Uint8Array {
// The synchronous signature. Every header here is computed in a straight
// line by `s3/sigv4.ts` — no promise, no `crypto.subtle` — because there is
// nowhere in this method to await one: it is called from an Emscripten
// stream `read` op that must return a byte count.
//
// `Range` is set *after* signing and is deliberately not part of the
// signature. S3 does not require it to be, and signing it would mean a
// fresh derivation for every chunk of every read rather than one per
// request against a cached key. The static headers — an API key, a bearer
// token — go on with it and before it, for the same reason and in the order
// `sources/access.ts` fixes.
const headers = { ...this.authorisation(), Range: `bytes=${from}-${to}` }
const answer = (this.syncFetch ?? xhrRange)(this.url, headers)
this.requests += 1
this.refuseStatus(answer.status, from, to)
this.bytesRead += answer.body.byteLength
this.onRead?.(this.requests, this.bytesRead)
return this.windowOf(answer.body, from, to, answer.contentRange)
}
/** What authorises a `GET` of this object — a signature, a header bag, both or neither. */
private authorisation(): Record<string, string> {
if (this.credential === undefined && this.headers === undefined) return {}
return accessHeaders({
url: this.url,
method: 'GET',
...(this.headers === undefined ? {} : { headers: this.headers }),
...(this.credential === undefined ? {} : { credential: this.credential }),
})
}
/** Refuse a ranged `GET` that was not answered with the range, in the words the fix needs. */
private refuseStatus(status: number, from: number, to: number): void {
if (status === 403 && this.credential !== undefined) {
// Distinguished from the range failure below because the fix is
// different and neither message would suggest the other. A signed request
// that is refused is a key problem — wrong region, expired token, an IAM
// policy that does not cover this prefix — not a host that ignores
// `Range`, and the reader has already got past the probe, so the object
// is there and readable in principle.
throw new ViewerError(
'bad-uri',
`${hostOf(this.url)} refused a signed request for bytes ${from}-${to} of ${this.url}. ` +
'The key reached the bucket and was rejected: check the region on the bucket entry ' +
'matches the bucket, that the key is allowed `s3:GetObject` on this prefix, and — ' +
'for a temporary credential — that the session token has not expired.',
{ url: this.url, status: 403, from, to, signed: true },
)
}
if (status !== 206 && status !== 200) {
throw new ViewerError(
'remote-no-ranges',
`${hostOf(this.url)} returned ${status} for bytes ${from}-${to} of ${this.url}`,
{ url: this.url, status, from, to },
)
}
}
/**
* Reduce a response body to exactly the window that was asked for.
*
* A host can ignore `Range` in two ways. The obvious one is a 200 with the
* whole object. The dangerous one is a **206** whose `Content-Range` covers
* the whole object — vite's own dev server does this, as do several CDNs.
* The status says "partial" and the first chunk is genuinely correct, so
* nothing looks wrong until a later read returns bytes from the *start* of
* the file while the caller believes they came from `from`. HDF5 then reads
* structure out of noise and fails hundreds of lines away, as a missing
* column or an unreadable attribute, with nothing pointing back here.
*
* `Content-Range` is believed when it is present, since it is the server's
* own statement of what it sent. Without it, a body longer than the window
* can only mean the whole object, which starts at zero.
*
* The third way is a body that does not match `Content-Range` *at all*, and
* it is caught first because believing the header is the whole strategy
* above. A 206 body is exactly its own range, so a disagreement means
* something between the server and here re-encoded it — transfer compression
* the probe in {@link RemoteRangeSource.open} did not see, because not every
* browser exposes `Content-Encoding` on a `fetch` response. Truncating the
* body to the window, which is what the arithmetic below would do, is exactly
* the silent corruption this class exists to refuse.
*/
private windowOf(
body: Uint8Array,
from: number,
to: number,
contentRange: string | null,
): Uint8Array {
const wanted = to - from + 1
const claimed = parseContentRange(contentRange)
if (claimed !== null && claimed.end - claimed.start + 1 !== body.byteLength) {
throw new ViewerError(
'remote-compressed',
`${hostOf(this.url)} answered a request for bytes ${from}-${to} of ${this.url} with ` +
`“Content-Range: ${contentRange}” — ${claimed.end - claimed.start + 1} octets — but ` +
`the body is ${body.byteLength}. The response was re-encoded in transit (transfer ` +
'compression), so ranges do not address the file. Serve it without ' +
'“Content-Encoding”, or download it and drop it here.',
{
url: this.url,
from,
to,
claimed: claimed.end - claimed.start + 1,
received: body.byteLength,
},
)
}
const start = claimed?.start ?? rangeStart(contentRange)
const skip = start === null ? (body.byteLength > wanted ? from : 0) : from - start
if (skip === 0 && body.byteLength <= wanted) return body
if (skip < 0 || skip >= body.byteLength) {
throw new ViewerError(
'remote-no-ranges',
`${hostOf(this.url)} answered a request for bytes ${from}-${to} of ${this.url} with ` +
`${body.byteLength} bytes starting at ${start ?? 'an unstated offset'}, which does ` +
'not contain what was asked for.',
{ url: this.url, from, to, start, length: body.byteLength },
)
}
return body.subarray(skip, Math.min(skip + wanted, body.byteLength))
}
private store(index: number, chunk: Uint8Array): void {
// Copy out of the response buffer so the (much larger) response can be
// collected once its useful window has been cached.
const owned = new Uint8Array(chunk)
this.cache.set(index, owned)
this.cachedBytes += owned.byteLength
for (const key of this.cache.keys()) {
if (this.cachedBytes <= this.maxCacheBytes) break
if (key === index) continue // never evict what was just stored
const evicted = this.cache.get(key)
if (evicted) this.cachedBytes -= evicted.byteLength
this.cache.delete(key)
}
}
/** Mark a cached chunk most-recently-used, by reinserting it at the back. */
private touch(index: number): void {
const chunk = this.cache.get(index)
if (!chunk) return
this.cache.delete(index)
this.cache.set(index, chunk)
}
}
/**
* Copy at most `limit` bytes of `source` from `at` into `target` at `into`,
* returning how many were written.
*
* A short response would otherwise leave the caller's loop unable to advance,
* so an empty copy is reported as a truncated read rather than spun on.
*/
function blit(
target: Uint8Array,
into: number,
source: Uint8Array,
at: number,
limit: number,
): number {
const n = Math.min(source.length - at, limit)
if (n <= 0) {
throw new ViewerError('remote-no-ranges', 'a range request returned a short response', {
wanted: limit,
available: source.length - at,
})
}
target.set(source.subarray(at, at + n), into)
return n
}
/**
* Turn an opaque `fetch` rejection into a refusal that names the host and says
* which of the two things went wrong.
*
* Both carry the code `remote-cors`, because that is the only code the shared
* {@link ErrorCode} union has for a request that never produced a response;
* `detail.reachable` is what a caller switches on to tell them apart without
* matching on prose.
*/
async function failCrossOrigin(
url: string,
host: string,
cause: unknown,
doFetch: typeof fetch,
cost: SourceStats,
): Promise<never> {
let reachable = false
try {
cost.requests += 1
await doFetch(url, { method: 'GET', mode: 'no-cors', headers: { Range: 'bytes=0-0' } })
reachable = true
} catch {
reachable = false
}
const detail = cause instanceof Error ? `${cause.name}: ${cause.message}` : String(cause)
if (reachable) {
throw new ViewerError(
'remote-cors',
`${host} is reachable but refused a cross-origin read of ${url} (${detail}). ` +
'That is what a missing “Access-Control-Allow-Origin” looks like from a browser: ' +
'the failure has no status and no headers. Configure the bucket to allow GET and ' +
'HEAD and the “Range” request header, and to expose “Content-Length” and ' +
'“Content-Range”.',
{ url, host, cause: detail, reachable: true },
)
}
throw new ViewerError(
'remote-cors',
`${host} could not be reached for ${url} (${detail}). The request failed before any ` +
'response arrived, so this is a DNS, TLS or connectivity problem rather than a ' +
'CORS one.',
{ url, host, cause: detail, reachable: false },
)
}
function hostOf(url: string): string {
try {
return new URL(url).host
} catch {
return url
}
}
/**
* The `Content-Encoding` a response was re-encoded with, or `null` when it was
* not — which includes the explicit `identity` and the empty header.
*
* A browser decodes the body transparently and leaves the header in place, so
* its presence is the signal; it is not asking us to decode anything.
*/
/**
* A `HEAD` for the whole file *as a range*, or `null` where the host would not
* answer one. Only ever sent after an unranged `HEAD` came back re-encoded —
* see the caller — so a host that never does that never sees it.
*/
async function rangedHead(
url: string,
doFetch: typeof fetch,
sign: (method: string) => Record<string, string>,
cost: SourceStats,
): Promise<Response | null> {
try {
cost.requests += 1
return await doFetch(url, { method: 'HEAD', headers: { ...sign('HEAD'), Range: 'bytes=0-' } })
} catch {
return null
}
}
function reEncoding(response: Response | null): string | null {
const raw = (response?.headers.get('content-encoding') ?? '').trim().toLowerCase()
if (raw === '' || raw === 'identity') return null
return raw
}
function numberHeader(response: Response | null, name: string): number | null {
const raw = response?.headers.get(name)
if (raw === null || raw === undefined || raw.trim() === '') return null
const value = Number(raw)
return Number.isFinite(value) ? value : null
}
/** `bytes 128-255/26520008` → `128`; `null` when the header is absent or odd. */
function rangeStart(header: string | null): number | null {
return parseContentRange(header)?.start ?? null
}
/**
* `bytes 128-255/26520008` → `{ start: 128, end: 255, total: 26520008 }`.
*
* The **span** is what {@link RemoteRangeSource.windowOf} needs and neither of
* the older two accessors exposed: it is the server's own statement of how many
* octets it put in the body, and a body that disagrees with it did not come off
* the wire the way the server sent it.
*/
function parseContentRange(
header: string | null,
): { start: number; end: number; total: number | null } | null {
if (!header) return null
const match = /bytes\s+(\d+)-(\d+)\/\s*(\d+|\*)\s*$/.exec(header)
if (!match) return null
const start = Number(match[1])
const end = Number(match[2])
const total = match[3] === '*' ? null : Number(match[3])
if (!Number.isFinite(start) || !Number.isFinite(end)) return null
return { start, end, total: total !== null && Number.isFinite(total) ? total : null }
}
/** `bytes 0-0/26520008` → `26520008`. */
function totalFromContentRange(header: string | null): number | null {
if (!header) return null
const match = /\/\s*(\d+)\s*$/.exec(header)
if (!match) return null
const value = Number(match[1])
return Number.isFinite(value) ? value : null
}
function hash(text: string): number {
let h = 0
for (let i = 0; i < text.length; i++) h = (Math.imul(31, h) + text.charCodeAt(i)) | 0
return h
}
|