io.c, 抽象了 avro_reader_t / avro_writer_t 出来,支持 read buffer from avro_reader_t 和 write buffer to avro_writer_t,目前支持 file/memory 两种 reader/writer。
datum_write.c datum_read.c, 支持 datum 对于 writer/reader 的写入/读出
schema.c, 支持 schema 对于 writer/reader 的写入/读出
datafile.c, file object container 实际上是将 schema 放入 file header,每次读取时,自动先读 schema,后解出数据(datum)。
encoding.h, 定义了 datum 的 en/decoder interface
encoding_binary.c, 实现了 en/decoder interface,支持序列化为 binary
schema 用于描述 datum 的结构(有哪些fields)
schema 支持 read from json 和 write to json,json 都是 pure text。
/- datum1
schema -- datum2
\- ...
datum 在传输过程中,序列化为 binary format。写入文件,也使用 binary format。
process A === datum binary(encoding.h) ===> process B
所以 encoding.h 定义了一个通用的 en/decoder interface。被 datafile.c 用来写 schema binary format,也被 datum 用来生成 binary format。
这几个东东之间的关系如下:
/*
* io
*/
avro_reader_t avro_reader_file(FILE * fp);
avro_writer_t avro_writer_file(FILE * fp);
avro_reader_t avro_reader_memory(const char *buf, int64_t len);
avro_writer_t avro_writer_memory(const char *buf, int64_t len);
int avro_read(avro_reader_t reader, void *buf, int64_t len);
int avro_skip(avro_reader_t reader, int64_t len);
int avro_write(avro_writer_t writer, void *buf, int64_t len);
void avro_writer_reset(avro_writer_t writer);
int64_t avro_writer_tell(avro_writer_t writer);
void avro_writer_flush(avro_writer_t writer);
void avro_writer_dump(avro_writer_t writer, FILE * fp);
void avro_reader_dump(avro_reader_t reader, FILE * fp);
void avro_reader_free(avro_reader_t reader);
void avro_writer_free(avro_writer_t writer);
/*
* datum
*/
int avro_read_data(avro_reader_t reader,
avro_schema_t writer_schema,
avro_schema_t reader_schema, avro_datum_t * datum);
int avro_skip_data(avro_reader_t reader, avro_schema_t writer_schema);
int avro_write_data(avro_writer_t writer,
avro_schema_t writer_schema, avro_datum_t datum);
int64_t avro_size_data(avro_writer_t writer,
avro_schema_t writer_schema, avro_datum_t datum);
/*
* schema
*/
int avro_schema_from_json(const char *jsontext,
const int32_t len,
avro_schema_t * schema, avro_schema_error_t * error);
int avro_schema_to_json(avro_schema_t schema, avro_writer_t out);
/*
* File object container
*/
typedef struct avro_file_reader_t_ *avro_file_reader_t;
typedef struct avro_file_writer_t_ *avro_file_writer_t;
int avro_file_writer_create(const char *path, avro_schema_t schema,
avro_file_writer_t * writer);
int avro_file_writer_open(const char *path, avro_file_writer_t * writer);
int avro_file_reader(const char *path, avro_file_reader_t * reader);
int avro_file_writer_append(avro_file_writer_t writer, avro_datum_t datum);
int avro_file_writer_sync(avro_file_writer_t writer);
int avro_file_writer_flush(avro_file_writer_t writer);
int avro_file_writer_close(avro_file_writer_t writer);
int avro_file_reader_read(avro_file_reader_t reader,
avro_schema_t readers_schema, avro_datum_t * datum);
int avro_file_reader_close(avro_file_reader_t reader);
评论