]> git.ipfire.org Git - thirdparty/FORT-validator.git/commit
New cache design
authorAlberto Leiva Popper <ydahhrk@gmail.com>
Mon, 7 Oct 2024 21:58:23 +0000 (15:58 -0600)
committerAlberto Leiva Popper <ydahhrk@gmail.com>
Mon, 7 Oct 2024 21:58:23 +0000 (15:58 -0600)
commitc7be17b62db6262e0bc9e33b7b9ad049b8563453
treed3223527bf2f9b9690c6912df5d4ce6cb8cbc240
parent447f261d4f9c8240bc73115e489c848f676158df
New cache design

For #82.

It seems I'm finally done making dramatic wide-reaching changes to the
codebase. There's still plenty to add and test, but I would like to
start pushing atomic commits from now on.

This is a squashed version of development brach "issue82". It includes
a few merges with main.

```
cache/
    rsync/              # rsync refresh
        0/              # rsync module: rsync://a.b.c/mod1
            rpp1/       # Repository Publication Point 1
                d.mft
                d.crl
                d1.roa
            rpp2/
                e.mft
                e.crl
            ta.cer      # Trust Anchor
        1/              # rsync module: rsync://a.b.c/mod2
            ...
        2/              # rsync module: rsync://x.y.z/mod1
            ...
    https/              # HTTPS refresh
        0               # https://a.b.c/repo/ta.cer
        1               # https://x.y.z/repo/ta.cer
    rrdp/               # RRDP refresh
        0/              # https://m.n.o/notification.xml
            0           # rsync://m.n.o/mod1/rpp1/a.mft
            1           # rsync://m.n.o/mod1/rpp1/b.crl
            2           # rsync://m.n.o/mod1/rpp2/c.mft
            3           # rsync://m.n.o/mod1/rpp2/d.cer
        1/              # https://p.q.r/notification.xml
            ...
    fallback/           # Committed RPPs
        0/              # Fallback of rsync://a.b.c/mod1/rpp1
            0           # Hard link: cache/rsync/0/rpp1/d.mft
            1           # Hard link: cache/rsync/0/rpp1/d.crl
            2           # rsync://a.b.c/mod1/rpp1/d2.roa
                        # (Unique because of refresh)
        1               # Hard link: cache/https/0
        2/              # Fallback of m.n.o's rsync://m.n.o/mod1/rpp1
            0           # Hard link: cache/rrdp/0/0
            1           # Hard link: cache/rrdp/0/1
    index.json          # URL/path mappings and some metadata
```

- `cache/rsync`, `cache/https` and `cache/rrdp` contain "refreshes"
  (the exact latest files according to the servers). RRDP withdraws are
  honored, and rsyncs run without --compare-dest.
- "Refresh" files marked as valid are backed up in `cache/fallback`
  at the end of each validation cycle.
- Validation first tests fallback+refresh. (If a file exists in both,
  refresh wins.) If that fails, it retries with fallback only.
- The index is not a tree; everything is caged in numbered directories
  and indexed by exact URL, to prevent file overriding by URL hacking.

There's also a `cache/tmp` directory, where Fort temporarily dumps
notifications, snapshots and deltas. This directory will be removed
once #127 is fixed.
164 files changed:
Makefile.am
docs/CVE.md [new file with mode: 0644]
docs/_layouts/default.html
docs/usage.md
examples/config.json
man/fort.8
src/Makefile.am
src/abbreviations.txt
src/asn1/decode.c
src/asn1/decode.h
src/asn1/oid.c
src/asn1/oid.h
src/asn1/signed_data.c
src/asn1/signed_data.h
src/base64.c [moved from src/crypto/base64.c with 96% similarity]
src/base64.h [moved from src/crypto/base64.h with 100% similarity]
src/cache.c [new file with mode: 0644]
src/cache.h [new file with mode: 0644]
src/cache/local_cache.c [deleted file]
src/cache/local_cache.h [deleted file]
src/cachetmp [new file with mode: 0644]
src/cachetmp.c [new file with mode: 0644]
src/cachetmp.h [new file with mode: 0644]
src/cert_stack.c [deleted file]
src/cert_stack.h [deleted file]
src/certificate_refs.c
src/certificate_refs.h
src/common.c
src/common.h
src/config.c
src/config.h
src/config/incidences.c
src/config/string_array.c
src/config/time.c [new file with mode: 0644]
src/config/time.h [new file with mode: 0644]
src/config/work_offline.h
src/extension.c
src/file.c
src/file.h
src/hash.c [moved from src/crypto/hash.c with 87% similarity]
src/hash.h [moved from src/crypto/hash.h with 73% similarity]
src/http.c [moved from src/http/http.c with 94% similarity]
src/http.h [new file with mode: 0644]
src/http/http.h [deleted file]
src/incidence.c [moved from src/incidence/incidence.c with 94% similarity]
src/incidence.h [moved from src/incidence/incidence.h with 85% similarity]
src/init.c
src/json_handler.c
src/json_util.c
src/json_util.h
src/libcrypto_util.c
src/libcrypto_util.h
src/log.c
src/log.h
src/main.c
src/object/bgpsec.h
src/object/certificate.c
src/object/certificate.h
src/object/crl.c
src/object/crl.h
src/object/ghostbusters.c
src/object/ghostbusters.h
src/object/manifest.c
src/object/manifest.h
src/object/roa.c
src/object/roa.h
src/object/signed_object.c
src/object/signed_object.h
src/object/tal.c
src/object/tal.h
src/output_printer.c
src/print_file.c
src/relax_ng.c [moved from src/xml/relax_ng.c with 93% similarity]
src/relax_ng.h [moved from src/xml/relax_ng.h with 98% similarity]
src/resource.c
src/resource.h
src/resource/asn.c
src/resource/asn.h
src/resource/ip4.c
src/resource/ip6.c
src/rpp.c [deleted file]
src/rpp.h [deleted file]
src/rrdp.c
src/rrdp.h
src/rsync.c [moved from src/rsync/rsync.c with 50% similarity]
src/rsync.h [moved from src/rsync/rsync.h with 53% similarity]
src/rtr/db/db_table.c
src/rtr/db/db_table.h
src/rtr/db/delta.c
src/rtr/db/delta.h
src/rtr/db/deltas_array.c
src/rtr/db/vrps.c
src/rtr/db/vrps.h
src/rtr/err_pdu.c
src/rtr/pdu.h
src/rtr/pdu_handler.c
src/rtr/pdu_sender.c
src/rtr/pdu_sender.h
src/rtr/pdu_stream.c
src/rtr/pdu_stream.h
src/rtr/primitive_writer.h
src/rtr/rtr.c
src/rtr/rtr.h
src/slurm/db_slurm.c
src/slurm/db_slurm.h
src/slurm/slurm_loader.c
src/slurm/slurm_parser.c
src/state.c
src/state.h
src/thread_pool.c [moved from src/thread/thread_pool.c with 99% similarity]
src/thread_pool.h [moved from src/thread/thread_pool.h with 100% similarity]
src/thread_var.c
src/thread_var.h
src/types/address.h
src/types/array.h [moved from src/data_structure/common.h with 54% similarity]
src/types/arraylist.h [moved from src/data_structure/array_list.h with 91% similarity]
src/types/asn.h [moved from src/as_number.h with 63% similarity]
src/types/bio_seq.h
src/types/map.c
src/types/map.h
src/types/name.c [moved from src/object/name.c with 90% similarity]
src/types/name.h [moved from src/object/name.h with 79% similarity]
src/types/path.c [moved from src/data_structure/path_builder.c with 69% similarity]
src/types/path.h [moved from src/data_structure/path_builder.h with 50% similarity]
src/types/rpp.c [new file with mode: 0644]
src/types/rpp.h [new file with mode: 0644]
src/types/sorted_array.c [moved from src/sorted_array.c with 99% similarity]
src/types/sorted_array.h [moved from src/sorted_array.h with 92% similarity]
src/types/str.c [moved from src/str_token.c with 79% similarity]
src/types/str.h [moved from src/str_token.h with 75% similarity]
src/types/url.c [new file with mode: 0644]
src/types/url.h [new file with mode: 0644]
src/types/uthash.h [moved from src/data_structure/uthash.h with 99% similarity]
src/types/vrp.h
src/validation_handler.h
test/Makefile.am
test/base64_test.c [moved from test/crypto/base64_test.c with 99% similarity]
test/cache/local_cache_test.c [deleted file]
test/cache_test.c [new file with mode: 0644]
test/cache_util.c [new file with mode: 0644]
test/cache_util.h [new file with mode: 0644]
test/common_test.c [moved from test/json_util_test.c with 63% similarity]
test/hash_test.c [moved from test/crypto/hash_test.c with 80% similarity]
test/mock.c
test/mock_https.c [new file with mode: 0644]
test/object/manifest_test.c [new file with mode: 0644]
test/object/tal_test.c [moved from test/tal_test.c with 73% similarity]
test/resources/rrdp/notif-bad-uri-4.xml [new file with mode: 0644]
test/resources/tal/4urls-ignored-protos.tal [new file with mode: 0644]
test/resources/xml/notification.xml [moved from test/xml/notification.xml with 100% similarity]
test/rrdp_test.c
test/rrdp_update_test.c [new file with mode: 0644]
test/rrdp_util.h [new file with mode: 0644]
test/rsync_test.c [new file with mode: 0644]
test/rtr/db/deltas_array_test.c
test/rtr/db/vrps_test.c
test/rtr/pdu_handler_test.c
test/rtr/pdu_stream_test.c
test/thread_pool_test.c
test/types/map_test.c [deleted file]
test/types/path_test.c [moved from test/data_structure/path_builder_test.c with 99% similarity]
test/types/url_test.c [new file with mode: 0644]
test/types/uthash_test.c [moved from test/data_structure/uthash_test.c with 99% similarity]
test/xml_test.c