Fijxu
/
nginx-quic
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge of r4985, r4986, r4987, r4988, r4989, r5002: access_log gzip. 

*) Access log: fixed redundant buffer reallocation.  Previously a new
   buffer was allocated for every "access_log" directive with the same
   file path and "buffer=" parameters, while only one buffer per file
   is used.

*) Reopening log files code moved to a separate function.  The code
   refactored in a way to call custom handler that can do appropriate
   cleanup work (if any), like flushing buffers, finishing compress
   streams, finalizing connections to log daemon, etc..

*) Access log: the "flush" parameter of the "access_log" directive.

*) Configure: added the NGX_ZLIB define.  This was introduced for
   conditional compilation of the code that requires the zlib library.

*) Access log: the "gzip" parameter of the "access_log" directive.
   Note: this requires zlib version 1.2.0.4 or above to work.

*) The data pointer in ngx_open_file_t objects must be initialized.
   Uninitialized pointer may result in arbitrary segfaults if access_log
   is used without buffer and without variables in file path.
   Patch by Tatsuhiko Kubo (ticket #268).
This commit is contained in:
Maxim Dounin 2013-02-11 14:34:00 +00:00
parent 203cbb1be3
commit 7cc223cc54
5 changed files with 391 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
auto/lib/zlib/conf
View File

@ -9,11 +9,13 @@ if [ $ZLIB != NONE ]; then
case "$NGX_CC_NAME" in
msvc* | owc* | bcc)
have=NGX_ZLIB . auto/have
LINK_DEPS="$LINK_DEPS $ZLIB/zlib.lib"
CORE_LIBS="$CORE_LIBS $ZLIB/zlib.lib"
;;
icc*)
have=NGX_ZLIB . auto/have
LINK_DEPS="$LINK_DEPS $ZLIB/libz.a"
# to allow -ipo optimization we link with the *.o but not library
@ -30,6 +32,7 @@ if [ $ZLIB != NONE ]; then
;;
*)
have=NGX_ZLIB . auto/have
LINK_DEPS="$LINK_DEPS $ZLIB/libz.a"
CORE_LIBS="$CORE_LIBS $ZLIB/libz.a"
#CORE_LIBS="$CORE_LIBS -L $ZLIB -lz"
@ -45,7 +48,7 @@ else
# FreeBSD, Solaris, Linux
ngx_feature="zlib library"
ngx_feature_name=
ngx_feature_name="NGX_ZLIB"
ngx_feature_run=no
ngx_feature_incs="#include <zlib.h>"
ngx_feature_path=

23
src/core/ngx_conf_file.c
View File

@ -945,7 +945,8 @@ ngx_conf_open_file(ngx_cycle_t *cycle, ngx_str_t *name)
file->name = *name;
}
file->buffer = NULL;
file->flush = NULL;
file->data = NULL;
return file;
}
@ -954,7 +955,6 @@ ngx_conf_open_file(ngx_cycle_t *cycle, ngx_str_t *name)
static void
ngx_conf_flush_files(ngx_cycle_t *cycle)
{
ssize_t n, len;
ngx_uint_t i;
ngx_list_part_t *part;
ngx_open_file_t *file;
@ -975,23 +975,8 @@ ngx_conf_flush_files(ngx_cycle_t *cycle)
i = 0;
}
len = file[i].pos - file[i].buffer;
if (file[i].buffer == NULL || len == 0) {
continue;
}
n = ngx_write_fd(file[i].fd, file[i].buffer, len);
if (n == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
ngx_write_fd_n " to \"%s\" failed",
file[i].name.data);
} else if (n != len) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
ngx_write_fd_n " to \"%s\" was incomplete: %z of %uz",
file[i].name.data, n, len);
if (file[i].flush) {
file[i].flush(&file[i], cycle->log);
}
}
}

11
src/core/ngx_conf_file.h
View File

@ -91,17 +91,8 @@ struct ngx_open_file_s {
ngx_fd_t fd;
ngx_str_t name;
u_char *buffer;
u_char *pos;
u_char *last;
#if 0
/* e.g. append mode, error_log */
ngx_uint_t flags;
/* e.g. reopen db file */
ngx_uint_t (*handler)(void *data, ngx_open_file_t *file);
void (*flush)(ngx_open_file_t *file, ngx_log_t *log);
void *data;
#endif
};

21
src/core/ngx_cycle.c
View File

@ -1115,7 +1115,6 @@ ngx_test_lockfile(u_char *file, ngx_log_t *log)
void
ngx_reopen_files(ngx_cycle_t *cycle, ngx_uid_t user)
{
ssize_t n, len;
ngx_fd_t fd;
ngx_uint_t i;
ngx_list_part_t *part;
@ -1139,24 +1138,8 @@ ngx_reopen_files(ngx_cycle_t *cycle, ngx_uid_t user)
continue;
}
len = file[i].pos - file[i].buffer;
if (file[i].buffer && len != 0) {
n = ngx_write_fd(file[i].fd, file[i].buffer, len);
if (n == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
ngx_write_fd_n " to \"%s\" failed",
file[i].name.data);
} else if (n != len) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
ngx_write_fd_n " to \"%s\" was incomplete: %z of %uz",
file[i].name.data, n, len);
}
file[i].pos = file[i].buffer;
if (file[i].flush) {
file[i].flush(&file[i], cycle->log);
}
fd = ngx_open_file(file[i].name.data, NGX_FILE_APPEND,

422
src/http/modules/ngx_http_log_module.c
View File

@ -9,6 +9,10 @@
#include <ngx_core.h>
#include <ngx_http.h>
#if (NGX_ZLIB)
#include <zlib.h>
#endif
typedef struct ngx_http_log_op_s ngx_http_log_op_t;
@ -40,6 +44,17 @@ typedef struct {
} ngx_http_log_main_conf_t;
typedef struct {
u_char *start;
u_char *pos;
u_char *last;
ngx_event_t *event;
ngx_msec_t flush;
ngx_int_t gzip;
} ngx_http_log_buf_t;
typedef struct {
ngx_array_t *lengths;
ngx_array_t *values;
@ -78,6 +93,17 @@ static void ngx_http_log_write(ngx_http_request_t *r, ngx_http_log_t *log,
static ssize_t ngx_http_log_script_write(ngx_http_request_t *r,
ngx_http_log_script_t *script, u_char **name, u_char *buf, size_t len);
#if (NGX_ZLIB)
static ssize_t ngx_http_log_gzip(ngx_fd_t fd, u_char *buf, size_t len,
ngx_int_t level, ngx_log_t *log);
static void *ngx_http_log_gzip_alloc(void *opaque, u_int items, u_int size);
static void ngx_http_log_gzip_free(void *opaque, void *address);
#endif
static void ngx_http_log_flush(ngx_open_file_t *file, ngx_log_t *log);
static void ngx_http_log_flush_handler(ngx_event_t *ev);
static u_char *ngx_http_log_pipe(ngx_http_request_t *r, u_char *buf,
ngx_http_log_op_t *op);
static u_char *ngx_http_log_time(ngx_http_request_t *r, u_char *buf,
@ -132,7 +158,7 @@ static ngx_command_t ngx_http_log_commands[] = {
{ ngx_string("access_log"),
NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF
|NGX_HTTP_LMT_CONF|NGX_CONF_TAKE123,
|NGX_HTTP_LMT_CONF|NGX_CONF_1MORE,
ngx_http_log_set_log,
NGX_HTTP_LOC_CONF_OFFSET,
0,
@ -216,8 +242,8 @@ ngx_http_log_handler(ngx_http_request_t *r)
size_t len;
ngx_uint_t i, l;
ngx_http_log_t *log;
ngx_open_file_t *file;
ngx_http_log_op_t *op;
ngx_http_log_buf_t *buffer;
ngx_http_log_loc_conf_t *lcf;
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
@ -258,21 +284,25 @@ ngx_http_log_handler(ngx_http_request_t *r)
len += NGX_LINEFEED_SIZE;
file = log[l].file;
buffer = log[l].file ? log[l].file->data : NULL;
if (file && file->buffer) {
if (buffer) {
if (len > (size_t) (file->last - file->pos)) {
if (len > (size_t) (buffer->last - buffer->pos)) {
ngx_http_log_write(r, &log[l], file->buffer,
file->pos - file->buffer);
ngx_http_log_write(r, &log[l], buffer->start,
buffer->pos - buffer->start);
file->pos = file->buffer;
buffer->pos = buffer->start;
}
if (len <= (size_t) (file->last - file->pos)) {
if (len <= (size_t) (buffer->last - buffer->pos)) {
p = file->pos;
p = buffer->pos;
if (buffer->event && p == buffer->start) {
ngx_add_timer(buffer->event, buffer->flush);
}
for (i = 0; i < log[l].format->ops->nelts; i++) {
p = op[i].run(r, p, &op[i]);
@ -280,10 +310,14 @@ ngx_http_log_handler(ngx_http_request_t *r)
ngx_linefeed(p);
file->pos = p;
buffer->pos = p;
continue;
}
if (buffer->event && buffer->event->timer_set) {
ngx_del_timer(buffer->event);
}
}
line = ngx_pnalloc(r->pool, len);
@ -310,14 +344,29 @@ static void
ngx_http_log_write(ngx_http_request_t *r, ngx_http_log_t *log, u_char *buf,
size_t len)
{
u_char *name;
time_t now;
ssize_t n;
ngx_err_t err;
u_char *name;
time_t now;
ssize_t n;
ngx_err_t err;
#if (NGX_ZLIB)
ngx_http_log_buf_t *buffer;
#endif
if (log->script == NULL) {
name = log->file->name.data;
#if (NGX_ZLIB)
buffer = log->file->data;
if (buffer && buffer->gzip) {
n = ngx_http_log_gzip(log->file->fd, buf, len, buffer->gzip,
r->connection->log);
} else {
n = ngx_write_fd(log->file->fd, buf, len);
}
#else
n = ngx_write_fd(log->file->fd, buf, len);
#endif
} else {
name = NULL;
@ -465,6 +514,194 @@ ngx_http_log_script_write(ngx_http_request_t *r, ngx_http_log_script_t *script,
}
#if (NGX_ZLIB)
static ssize_t
ngx_http_log_gzip(ngx_fd_t fd, u_char *buf, size_t len, ngx_int_t level,
ngx_log_t *log)
{
int rc, wbits, memlevel;
u_char *out;
size_t size;
ssize_t n;
z_stream zstream;
ngx_err_t err;
ngx_pool_t *pool;
wbits = MAX_WBITS;
memlevel = MAX_MEM_LEVEL - 1;
while ((ssize_t) len < ((1 << (wbits - 1)) - 262)) {
wbits--;
memlevel--;
}
/*
* This is a formula from deflateBound() for conservative upper bound of
* compressed data plus 18 bytes of gzip wrapper.
*/
size = len + ((len + 7) >> 3) + ((len + 63) >> 6) + 5 + 18;
ngx_memzero(&zstream, sizeof(z_stream));
pool = ngx_create_pool(256, log);
if (pool == NULL) {
/* simulate successful logging */
return len;
}
pool->log = log;
zstream.zalloc = ngx_http_log_gzip_alloc;
zstream.zfree = ngx_http_log_gzip_free;
zstream.opaque = pool;
out = ngx_pnalloc(pool, size);
if (out == NULL) {
goto done;
}
zstream.next_in = buf;
zstream.avail_in = len;
zstream.next_out = out;
zstream.avail_out = size;
rc = deflateInit2(&zstream, (int) level, Z_DEFLATED, wbits + 16, memlevel,
Z_DEFAULT_STRATEGY);
if (rc != Z_OK) {
ngx_log_error(NGX_LOG_ALERT, log, 0, "deflateInit2() failed: %d", rc);
goto done;
}
ngx_log_debug4(NGX_LOG_DEBUG_HTTP, log, 0,
"deflate in: ni:%p no:%p ai:%ud ao:%ud",
zstream.next_in, zstream.next_out,
zstream.avail_in, zstream.avail_out);
rc = deflate(&zstream, Z_FINISH);
if (rc != Z_STREAM_END) {
ngx_log_error(NGX_LOG_ALERT, log, 0,
"deflate(Z_FINISH) failed: %d", rc);
goto done;
}
ngx_log_debug5(NGX_LOG_DEBUG_HTTP, log, 0,
"deflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d",
zstream.next_in, zstream.next_out,
zstream.avail_in, zstream.avail_out,
rc);
size -= zstream.avail_out;
rc = deflateEnd(&zstream);
if (rc != Z_OK) {
ngx_log_error(NGX_LOG_ALERT, log, 0, "deflateEnd() failed: %d", rc);
goto done;
}
n = ngx_write_fd(fd, out, size);
if (n != (ssize_t) size) {
err = (n == -1) ? ngx_errno : 0;
ngx_destroy_pool(pool);
ngx_set_errno(err);
return -1;
}
done:
ngx_destroy_pool(pool);
/* simulate successful logging */
return len;
}
static void *
ngx_http_log_gzip_alloc(void *opaque, u_int items, u_int size)
{
ngx_pool_t *pool = opaque;
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pool->log, 0,
"gzip alloc: n:%ud s:%ud", items, size);
return ngx_palloc(pool, items * size);
}
static void
ngx_http_log_gzip_free(void *opaque, void *address)
{
#if 0
ngx_pool_t *pool = opaque;
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pool->log, 0, "gzip free: %p", address);
#endif
}
#endif
static void
ngx_http_log_flush(ngx_open_file_t *file, ngx_log_t *log)
{
size_t len;
ssize_t n;
ngx_http_log_buf_t *buffer;
buffer = file->data;
len = buffer->pos - buffer->start;
if (len == 0) {
return;
}
#if (NGX_ZLIB)
if (buffer->gzip) {
n = ngx_http_log_gzip(file->fd, buffer->start, len, buffer->gzip, log);
} else {
n = ngx_write_fd(file->fd, buffer->start, len);
}
#else
n = ngx_write_fd(file->fd, buffer->start, len);
#endif
if (n == -1) {
ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
ngx_write_fd_n " to \"%s\" failed",
file->name.data);
} else if ((size_t) n != len) {
ngx_log_error(NGX_LOG_ALERT, log, 0,
ngx_write_fd_n " to \"%s\" was incomplete: %z of %uz",
file->name.data, n, len);
}
buffer->pos = buffer->start;
if (buffer->event && buffer->event->timer_set) {
ngx_del_timer(buffer->event);
}
}
static void
ngx_http_log_flush_handler(ngx_event_t *ev)
{
ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0,
"http log buffer flush handler");
ngx_http_log_flush(ev->data, ev->log);
}
static u_char *
ngx_http_log_copy_short(ngx_http_request_t *r, u_char *buf,
ngx_http_log_op_t *op)
@ -848,10 +1085,13 @@ ngx_http_log_set_log(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
{
ngx_http_log_loc_conf_t *llcf = conf;
ssize_t buf;
ssize_t size;
ngx_int_t gzip;
ngx_uint_t i, n;
ngx_str_t *value, name;
ngx_msec_t flush;
ngx_str_t *value, name, s;
ngx_http_log_t *log;
ngx_http_log_buf_t *buffer;
ngx_http_log_fmt_t *fmt;
ngx_http_log_main_conf_t *lmcf;
ngx_http_script_compile_t sc;
@ -936,54 +1176,152 @@ ngx_http_log_set_log(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
&& ngx_strcasecmp(fmt[i].name.data, name.data) == 0)
{
log->format = &fmt[i];
goto buffer;
break;
}
}
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"unknown log format \"%V\"", &name);
return NGX_CONF_ERROR;
if (log->format == NULL) {
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"unknown log format \"%V\"", &name);
return NGX_CONF_ERROR;
}
buffer:
size = 0;
flush = 0;
gzip = 0;
if (cf->args->nelts == 4) {
if (ngx_strncmp(value[3].data, "buffer=", 7) != 0) {
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"invalid parameter \"%V\"", &value[3]);
return NGX_CONF_ERROR;
for (i = 3; i < cf->args->nelts; i++) {
if (ngx_strncmp(value[i].data, "buffer=", 7) == 0) {
s.len = value[i].len - 7;
s.data = value[i].data + 7;
size = ngx_parse_size(&s);
if (size == NGX_ERROR || size == 0) {
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"invalid buffer size \"%V\"", &s);
return NGX_CONF_ERROR;
}
continue;
}
if (ngx_strncmp(value[i].data, "flush=", 6) == 0) {
s.len = value[i].len - 6;
s.data = value[i].data + 6;
flush = ngx_parse_time(&s, 0);
if (flush == (ngx_msec_t) NGX_ERROR || flush == 0) {
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"invalid flush time \"%V\"", &s);
return NGX_CONF_ERROR;
}
continue;
}
if (ngx_strncmp(value[i].data, "gzip", 4) == 0
&& (value[i].len == 4 || value[i].data[4] == '='))
{
#if (NGX_ZLIB)
if (size == 0) {
size = 64 * 1024;
}
if (value[i].len == 4) {
gzip = Z_BEST_SPEED;
continue;
}
s.len = value[i].len - 5;
s.data = value[i].data + 5;
gzip = ngx_atoi(s.data, s.len);
if (gzip < 1 || gzip > 9) {
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"invalid compression level \"%V\"", &s);
return NGX_CONF_ERROR;
}
continue;
#else
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"nginx was built without zlib support");
return NGX_CONF_ERROR;
#endif
}
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"invalid parameter \"%V\"", &value[i]);
return NGX_CONF_ERROR;
}
if (flush && size == 0) {
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"no buffer is defined for access_log \"%V\"",
&value[1]);
return NGX_CONF_ERROR;
}
if (size) {
if (log->script) {
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"buffered logs cannot have variables in name");
return NGX_CONF_ERROR;
}
name.len = value[3].len - 7;
name.data = value[3].data + 7;
if (log->file->data) {
buffer = log->file->data;
buf = ngx_parse_size(&name);
if (buffer->last - buffer->start != size
|| buffer->flush != flush
|| buffer->gzip != gzip)
{
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"access_log \"%V\" already defined "
"with conflicting parameters",
&value[1]);
return NGX_CONF_ERROR;
}
if (buf == NGX_ERROR) {
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"invalid buffer value \"%V\"", &name);
return NGX_CONF_OK;
}
buffer = ngx_pcalloc(cf->pool, sizeof(ngx_http_log_buf_t));
if (buffer == NULL) {
return NGX_CONF_ERROR;
}
if (log->file->buffer && log->file->last - log->file->pos != buf) {
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"access_log \"%V\" already defined "
"with different buffer size", &value[1]);
buffer->start = ngx_pnalloc(cf->pool, size);
if (buffer->start == NULL) {
return NGX_CONF_ERROR;
}
log->file->buffer = ngx_palloc(cf->pool, buf);
if (log->file->buffer == NULL) {
return NGX_CONF_ERROR;
buffer->pos = buffer->start;
buffer->last = buffer->start + size;
if (flush) {
buffer->event = ngx_pcalloc(cf->pool, sizeof(ngx_event_t));
if (buffer->event == NULL) {
return NGX_CONF_ERROR;
}
buffer->event->data = log->file;
buffer->event->handler = ngx_http_log_flush_handler;
buffer->event->log = &cf->cycle->new_log;
buffer->flush = flush;
}
log->file->pos = log->file->buffer;
log->file->last = log->file->buffer + buf;
buffer->gzip = gzip;
log->file->flush = ngx_http_log_flush;
log->file->data = buffer;
}
return NGX_CONF_OK;