From: Michael Schroeder Date: Mon, 10 Dec 2007 20:16:17 +0000 (+0000) Subject: - use schema in repodata to make it extendible X-Git-Tag: BASE-SuSE-Code-12_1-Branch~308^2~749 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ea25bc9ec4d37f749a567b75cb3e086614c23acd;p=thirdparty%2Flibsolv.git - use schema in repodata to make it extendible --- diff --git a/src/pool.c b/src/pool.c index 6e05c643..d23d76c3 100644 --- a/src/pool.c +++ b/src/pool.c @@ -57,6 +57,9 @@ static const char *initpool_data[] = { "src", "nosrc", "noarch", + "repodata:external", + "repodata:keys", + "repodata:location", 0 }; diff --git a/src/pool.h b/src/pool.h index 45e08134..4042a04c 100644 --- a/src/pool.h +++ b/src/pool.h @@ -50,6 +50,12 @@ extern "C" { #define ARCH_SRC 21 #define ARCH_NOSRC 22 #define ARCH_NOARCH 23 +#define REPODATA_EXTERNAL 24 +#define REPODATA_KEYS 25 +#define REPODATA_LOCATION 26 + +#define ID_NUM_INTERNAL 27 + /* well known solvable */ #define SYSTEMSOLVABLE 1 @@ -121,6 +127,7 @@ struct _Pool { #define SAT_DEBUG_JOB (1<<11) #define SAT_DEBUG_SCHUBI (1<<12) +#define TYPE_VOID 0 #define TYPE_ID 1 #define TYPE_IDARRAY 2 #define TYPE_STR 3 @@ -137,6 +144,9 @@ struct _Pool { #define TYPE_COUNTED 12 #define TYPE_ATTR_TYPE_MAX 12 +#define TYPE_IDVALUEARRAY 13 +#define TYPE_IDVALUEVALUEARRAY 14 + //----------------------------------------------- diff --git a/src/repo.c b/src/repo.c index 76d9e9fa..90deae54 100644 --- a/src/repo.c +++ b/src/repo.c @@ -554,16 +554,16 @@ key_cmp (const void *pa, const void *pb) } void -repo_add_attrstore (Repo *repo, Attrstore *s, const char *name) +repo_add_attrstore (Repo *repo, Attrstore *s, const char *location) { unsigned i; Repodata *data; /* If this is meant to be the embedded attributes, make sure we don't have them already. */ - if (!name) + if (!location) { for (i = 0; i < repo->nrepodata; i++) - if (repo->repodata[i].name == 0) + if (repo->repodata[i].location == 0) break; if (i != repo->nrepodata) { @@ -589,8 +589,8 @@ repo_add_attrstore (Repo *repo, Attrstore *s, const char *name) } qsort (data->keys, data->nkeys, sizeof (data->keys[0]), key_cmp); } - if (name) - data->name = strdup (name); + if (location) + data->location = strdup(location); } // EOF diff --git a/src/repo.h b/src/repo.h index 597287c2..eda14d20 100644 --- a/src/repo.h +++ b/src/repo.h @@ -27,15 +27,17 @@ typedef struct _Repodata { Id name; unsigned type; } *keys; - /* Length of names. */ + /* Length of keys array */ unsigned nkeys; + /* The attribute store itself. */ Attrstore *s; /* A filename where to find this attribute store, or where to store it. May be "", in which case we can't load it on demand or store into it. It may also be NULL for at most one of the repodata per repo, in which case these are the embedded attributes. */ - const char *name; + + const char *location; /* The SHA1 checksum of the file. */ unsigned char checksum[20]; } Repodata; @@ -70,7 +72,7 @@ extern Offset repo_addid_dep(Repo *repo, Offset olddeps, Id id, int isreq); extern Offset repo_reserve_ids(Repo *repo, Offset olddeps, int num); extern Offset repo_fix_legacy(Repo *repo, Offset provides, Offset supplements); -extern void repo_add_attrstore (Repo *repo, Attrstore *s, const char *name); +extern void repo_add_attrstore (Repo *repo, Attrstore *s, const char *location); static inline const char *repo_name(const Repo *repo) { diff --git a/src/repo_solv.c b/src/repo_solv.c index aebbb381..9f139d48 100644 --- a/src/repo_solv.c +++ b/src/repo_solv.c @@ -151,7 +151,7 @@ read_idarray(FILE *fp, Id max, Id *map, Id *store, Id *end, int relative) x = (x - 1) + old; old = x; } - if (x >= max) + if (max && x >= max) { pool_debug(mypool, SAT_FATAL, "read_idarray: id too large (%u/%u)\n", x, max); exit(1); @@ -221,10 +221,12 @@ read_str (FILE *fp, char **inbuf, unsigned *len) } static void -skip_item (FILE *fp, unsigned type, Id *idmap, unsigned numid, unsigned numrel) +skip_item (FILE *fp, unsigned type, unsigned numid, unsigned numrel) { switch (type) { + case TYPE_VOID: + break; case TYPE_ID: read_id(fp, numid + numrel); /* just check Id */ break; @@ -237,6 +239,8 @@ skip_item (FILE *fp, unsigned type, Id *idmap, unsigned numid, unsigned numrel) ; break; case TYPE_IDARRAY: + case TYPE_IDVALUEARRAY: + case TYPE_IDVALUEVALUEARRAY: case TYPE_REL_IDARRAY: while ((read_u8(fp) & 0xc0) != 0) ; @@ -248,7 +252,7 @@ skip_item (FILE *fp, unsigned type, Id *idmap, unsigned numid, unsigned numrel) { read_id (fp, numid); /* Name */ unsigned t = read_id (fp, TYPE_ATTR_TYPE_MAX + 1); - skip_item (fp, t, idmap, numid, numrel); + skip_item (fp, t, numid, numrel); } } break; @@ -257,7 +261,7 @@ skip_item (FILE *fp, unsigned type, Id *idmap, unsigned numid, unsigned numrel) unsigned count = read_id (fp, 0); unsigned t = read_id (fp, TYPE_ATTR_TYPE_MAX + 1); while (count--) - skip_item (fp, t, idmap, numid, numrel); + skip_item (fp, t, numid, numrel); } break; case TYPE_ATTR_CHUNK: @@ -286,87 +290,89 @@ key_cmp (const void *pa, const void *pb) return a->name - b->name; } +struct key { + Id name; + Id type; + Id size; +}; + static void -parse_repodata (FILE *fp, Id *idmap, unsigned numid, unsigned numrel, Repo *repo) +parse_repodata (FILE *fp, Id *keyp, struct key *keys, Id *idmap, unsigned numid, unsigned numrel, Repo *repo) { - unsigned count = read_id (fp, 0); - - while (count--) - { - Repodata *data; - read_id (fp, numid); /* no name */ - unsigned type = read_id (fp, TYPE_ATTR_TYPE_MAX + 1); - if (type != TYPE_COUNT_NAMED) - { - skip_item (fp, type, idmap, numid, numrel); - continue; - } - unsigned c = read_id (fp, 0); - if (c == 0) - continue; - if (c != 2) - { - pool_debug (mypool, SAT_FATAL, "invalid attribute data\n"); - exit (1); - } - read_id (fp, numid); /* no name */ - if (read_id (fp, TYPE_ATTR_TYPE_MAX + 1) != TYPE_STR) - { - pool_debug (mypool, SAT_FATAL, "invalid attribute data\n"); - exit (1); - } - char buf[1024]; - unsigned len = sizeof (buf); - char *filename = buf; - read_str (fp, &filename, &len); + Id key, id; + Id *ida, *ide; + Repodata *data; + int i, n; - read_id (fp, numid); /* no name */ - if (read_id (fp, TYPE_ATTR_TYPE_MAX + 1) != TYPE_COUNTED) - { - pool_debug (mypool, SAT_FATAL, "invalid attribute data\n"); - exit (1); - } + repo->repodata = xrealloc(repo->repodata, (repo->nrepodata + 1) * sizeof (*data)); + data = repo->repodata + repo->nrepodata++; + memset(data, 0, sizeof(*data)); - unsigned nkeys = read_id (fp, 0); - if ((nkeys & 1) != 0 - || read_id (fp, TYPE_ATTR_TYPE_MAX + 1) != TYPE_ID) - { - pool_debug (mypool, SAT_FATAL, "invalid attribute data\n"); - exit (1); + while ((key = *keyp++) != 0) + { + id = keys[key].name; + switch (keys[key].type) + { + case TYPE_IDVALUEARRAY: + if (id != REPODATA_KEYS) + { + skip_item(fp, TYPE_IDVALUEARRAY, numid, numrel); + break; + } + ida = xcalloc(keys[key].size, sizeof(Id)); + ide = read_idarray(fp, 0, 0, ida, ida + keys[key].size, 0); + n = ide - ida - 1; + if (n & 1) + { + pool_debug (mypool, SAT_FATAL, "invalid attribute data\n"); + exit (1); + } + data->nkeys = n >> 1; + data->keys = xmalloc2(data->nkeys, sizeof(data->keys[0])); + for (i = 0, ide = ida; i < data->nkeys; i++) + { + if (*ide >= numid) + { + pool_debug (mypool, SAT_FATAL, "invalid attribute data\n"); + exit (1); + } + data->keys[i].name = idmap[*ide++]; + data->keys[i].type = *ide++; + } + xfree(ida); + qsort(data->keys, data->nkeys, sizeof(data->keys[0]), key_cmp); + break; + case TYPE_STR: + if (id != REPODATA_LOCATION) + skip_item(fp, TYPE_STR, numid, numrel); + else + { + char buf[1024]; + unsigned len = sizeof (buf); + char *filename = buf; + read_str(fp, &filename, &len); + data->location = strdup(filename); + if (filename != buf) + free(filename); + } + break; + default: + skip_item(fp, keys[key].type, numid, numrel); + break; } - nkeys >>= 1; - - repo->nrepodata++; - data = xrealloc (repo->repodata, repo->nrepodata * sizeof (*data)); - repo->repodata = data; - data += repo->nrepodata - 1; - memset (data, 0, sizeof (*data)); - data->nkeys = nkeys; - if (data->nkeys) - { - unsigned i; - data->keys = xmalloc (data->nkeys * sizeof (data->keys[0])); - for (i = 0; i < data->nkeys; i++) - { - data->keys[i].name = idmap[read_id (fp, numid)]; - data->keys[i].type = read_id (fp, 0); - } - qsort (data->keys, data->nkeys, sizeof (data->keys[0]), key_cmp); - } - data->name = strdup (filename); - - if (filename != buf) - xfree (filename); } } /*-----------------------------------------------------------------*/ -struct key { - Id name; - Id type; - Id size; -}; + +void +skip_schema(FILE *fp, Id *keyp, struct key *keys, unsigned int numid, unsigned int numrel) +{ + Id key; + while ((key = *keyp++) != 0) + skip_item(fp, keys[key].type, numid, numrel); +} // ---------------------------------------------- @@ -692,16 +698,20 @@ repo_add_solv(Repo *repo, FILE *fp) } /******* Part 5: Info ***********************************************/ - /* we skip the info for now... */ for (i = 0; i < numinfo; i++) { - unsigned name = idmap[read_id (fp, numid)]; - unsigned type = read_id (fp, TYPE_ATTR_TYPE_MAX + 1); - if (type == TYPE_COUNT_NAMED - && !strcmp (id2str (pool, name), "repodata")) - parse_repodata (fp, idmap, numid, numrel, repo); + /* for now we're just interested in data that starts with + * the repodata_external id + */ + Id *keyp = schemadata + schemata[read_id(fp, numschemata)]; + key = *keyp; + if (keys[key].name == REPODATA_EXTERNAL && keys[key].type == TYPE_VOID) + { + /* external data for some ids */ + parse_repodata(fp, keyp, keys, idmap, numid, numrel, repo); + } else - skip_item (fp, type, idmap, numid, numrel); + skip_schema(fp, keyp, keys, numid, numrel); } /******* Part 6: packed sizes (optional) ****************************/ @@ -932,6 +942,8 @@ repo_add_solv(Repo *repo, FILE *fp) embedded_store = new_store (pool); add_attr_from_file (embedded_store, i, id, keys[key].type, idmap, numid, fp); break; + default: + skip_item(fp, keys[key].type, numid, numrel); } } } diff --git a/tools/dumpsolv.c b/tools/dumpsolv.c index c2b05f95..e120c485 100644 --- a/tools/dumpsolv.c +++ b/tools/dumpsolv.c @@ -89,19 +89,19 @@ static void dump_repodata (Repo *repo) { unsigned i; - Repodata *d; + Repodata *data; if (repo->nrepodata == 0) return; - printf ("repo refers to %d attribute stores:\n", repo->nrepodata); - for (i = 0, d = repo->repodata; i < repo->nrepodata; i++, d++) + printf("repo refers to %d attribute stores:\n", repo->nrepodata); + for (i = 0, data = repo->repodata; i < repo->nrepodata; i++, data++) { - unsigned j; - printf ("%s has %d keys", d->name ? d->name : "**EMBED**", d->nkeys); - for (j = 0; j < d->nkeys; j++) - printf ("\n %s", id2str (repo->pool, d->keys[j].name)); - printf ("\n"); + unsigned int j; + printf("%s has %d keys", data->location ? data->location : "**EMBED**", data->nkeys); + for (j = 0; j < data->nkeys; j++) + printf("\n %s", id2str(repo->pool, data->keys[j].name)); + printf("\n"); } - printf ("\n"); + printf("\n"); } static void diff --git a/tools/repo_write.c b/tools/repo_write.c index ef36e61e..a5abf3d5 100644 --- a/tools/repo_write.c +++ b/tools/repo_write.c @@ -320,39 +320,42 @@ repo_write(Repo *repo, FILE *fp) Reldep *ran; Id *idarraydata; - int idsizes[RPM_RPMDBID + 1]; - int id2key[RPM_RPMDBID + 1]; + int idsizes[ID_NUM_INTERNAL]; + int id2key[ID_NUM_INTERNAL]; int nsolvables; Id *schemadata, *schemadatap, *schema, *sp; Id schemaid; int schemadatalen; Id *solvschema; /* schema of our solvables */ + Id repodataschema, repodataschema_internal; Id lastschema[256]; Id lastschemakey[256]; - /* For the info block. */ - Id repodata_id, hello_id; - - repodata_id = str2id (pool, "repodata", 1); - hello_id = str2id (pool, "hello", 1); - nsolvables = 0; idarraydata = repo->idarraydata; needid = (NeedId *)xcalloc(pool->ss.nstrings + pool->nrels, sizeof(*needid)); + memset(idsizes, 0, sizeof(idsizes)); - needid[repodata_id].need++; - needid[hello_id].need++; + repodataschema = repodataschema_internal = 0; for (i = 0; i < repo->nrepodata; i++) { int j; + idsizes[REPODATA_EXTERNAL] = 1; + idsizes[REPODATA_KEYS]++; + if (repo->repodata[i].location) + { + repodataschema = 1; /* mark that we need it */ + idsizes[REPODATA_LOCATION] = 1; + } + else + repodataschema_internal = 1; /* mark that we need it */ for (j = 0; j < repo->repodata[i].nkeys; j++) needid[repo->repodata[i].keys[j].name].need++; + idsizes[REPODATA_KEYS] += 2 * repo->repodata[i].nkeys; } - memset(idsizes, 0, sizeof(idsizes)); - for (i = repo->start, s = pool->solvables + i; i < repo->end; i++, s++) { if (s->repo != repo) @@ -394,7 +397,7 @@ repo_write(Repo *repo, FILE *fp) if (repo->rpmdbid) idsizes[RPM_RPMDBID] = 1; - for (i = SOLVABLE_NAME; i <= RPM_RPMDBID; i++) + for (i = SOLVABLE_NAME; i < ID_NUM_INTERNAL; i++) { if (idsizes[i]) needid[i].need++; @@ -439,7 +442,7 @@ repo_write(Repo *repo, FILE *fp) /* find the keys we need */ nkeys = 1; memset(id2key, 0, sizeof(id2key)); - for (i = SOLVABLE_NAME; i <= RPM_RPMDBID; i++) + for (i = SOLVABLE_NAME; i < ID_NUM_INTERNAL; i++) if (idsizes[i]) id2key[i] = nkeys++; @@ -525,7 +528,26 @@ repo_write(Repo *repo, FILE *fp) } solvschema[n++] = schemaid; } - /* convert all schemas to keys */ + + if (repodataschema) + { + /* add us a schema for our repodata */ + repodataschema = nschemata++; + *schemadatap++ = REPODATA_EXTERNAL; + *schemadatap++ = REPODATA_KEYS; + *schemadatap++ = REPODATA_LOCATION; + *schemadatap++ = 0; + } + if (repodataschema_internal) + { + /* add us a schema for our repodata */ + repodataschema = nschemata++; + *schemadatap++ = REPODATA_EXTERNAL; + *schemadatap++ = REPODATA_KEYS; + *schemadatap++ = 0; + } + + /* convert all schemas to local keys */ for (sp = schemadata; sp < schemadatap; sp++) *sp = id2key[*sp]; @@ -539,7 +561,7 @@ repo_write(Repo *repo, FILE *fp) write_u32(fp, nsolvables); write_u32(fp, nkeys); write_u32(fp, nschemata); - write_u32(fp, 2); /* Info block. */ + write_u32(fp, repo->nrepodata); /* info blocks. */ solv_flags = 0; solv_flags |= SOLV_FLAG_PREFIX_POOL; #if 0 @@ -591,13 +613,13 @@ repo_write(Repo *repo, FILE *fp) ran = pool->rels + (needid[pool->ss.nstrings + i].map - pool->ss.nstrings); write_id(fp, needid[ISRELDEP(ran->name) ? RELOFF(ran->name) : ran->name].need); write_id(fp, needid[ISRELDEP(ran->evr) ? RELOFF(ran->evr) : ran->evr].need); - write_u8( fp, ran->flags); + write_u8(fp, ran->flags); } /* * write keys */ - for (i = SOLVABLE_NAME; i <= RPM_RPMDBID; i++) + for (i = SOLVABLE_NAME; i < ID_NUM_INTERNAL; i++) { if (!idsizes[i]) continue; @@ -606,6 +628,12 @@ repo_write(Repo *repo, FILE *fp) write_id(fp, TYPE_REL_IDARRAY); else if (i == RPM_RPMDBID) write_id(fp, TYPE_U32); + else if (i == REPODATA_EXTERNAL) + write_id(fp, TYPE_VOID); + else if (i == REPODATA_KEYS) + write_id(fp, TYPE_IDVALUEARRAY); + else if (i == REPODATA_LOCATION) + write_id(fp, TYPE_STR); else write_id(fp, TYPE_ID); write_id(fp, idsizes[i]); @@ -625,43 +653,29 @@ repo_write(Repo *repo, FILE *fp) /* * write info block */ - write_id (fp, needid[hello_id].need); - write_id (fp, TYPE_COUNT_NAMED); - write_id (fp, 1); - write_id (fp, 0); //name - write_id (fp, TYPE_STR); - write_str (fp, "doll"); - - write_id (fp, needid[repodata_id].need); - write_id (fp, TYPE_COUNT_NAMED); - write_id (fp, repo->nrepodata); for (i = 0; i < repo->nrepodata; i++) { int j; - write_id (fp, 0); /* no name, isn't important here */ - write_id (fp, TYPE_COUNT_NAMED); - /* Don't emit the embedded attributes. */ - if (repo->repodata[i].name == 0) + + if (repo->repodata[i].location) + write_id(fp, repodataschema); + else + write_id(fp, repodataschema_internal); + /* keys + location, write idarray */ + for (j = 0; j < repo->repodata[i].nkeys; j++) { - write_id (fp, 0); /* count */ - continue; - } - write_id (fp, 2); /* 2 items, the filename and the keys */ - /* 1 filename */ - write_id (fp, 0); /* no name */ - write_id (fp, TYPE_STR); - write_str (fp, repo->repodata[i].name); - - /* 2 keys */ - write_id (fp, 0); /* no name */ - write_id (fp, TYPE_COUNTED); - write_id (fp, repo->repodata[i].nkeys * 2); - write_id (fp, TYPE_ID); - for (j = 0; j < repo->repodata[i].nkeys; j++) - { - write_id (fp, needid[repo->repodata[i].keys[j].name].need); - write_id (fp, repo->repodata[i].keys[j].type); - } + /* this looks horrible, we need some function */ + Id id = needid[repo->repodata[i].keys[j].name].need; + if (id >= 64) + id = (id & 63) | ((id & ~63) << 1); + write_id(fp, id | 0x40); + id = repo->repodata[i].keys[j].type; + if (id >= 64) + id = (id & 63) | ((id & ~63) << 1); + write_id(fp, id | (j < repo->repodata[i].nkeys - 1 ? 0x40: 0)); + } + if (repo->repodata[i].location) + write_str(fp, repo->repodata[i].location); } #if 0