[{"data":1,"prerenderedAt":800},["ShallowReactive",2],{"mdc-rpekmd-key":3,"mdc-43zvw-key":28,"mdc-1jk1y2-key":271,"mdc--yzg5rr-key":305,"mdc--l070by-key":317,"mdc-r7cnr6-key":395,"mdc-adzzkz-key":426,"mdc--6xddsi-key":479,"mdc-r12m8s-key":489,"mdc-qyold8-key":616,"mdc--cun6br-key":628},{"data":4,"body":5},{},{"type":6,"children":7},"root",[8,23],{"type":9,"tag":10,"props":11,"children":12},"element","p",{},[13,21],{"type":9,"tag":14,"props":15,"children":17},"code",{"className":16},[],[18],{"type":19,"value":20},"text","docker logs",{"type":19,"value":22}," works beautifully for one container on one host. It stops working the moment there are thirty containers across six hosts and the one you need died an hour ago.",{"type":9,"tag":10,"props":24,"children":25},{},[26],{"type":19,"value":27},"Production observability for containers is three separate questions with three separate answers: what did it print, what did the daemon do, and what is it consuming.",{"data":29,"body":30},{},{"type":6,"children":31},[32,39,44,236,249],{"type":9,"tag":33,"props":34,"children":36},"h2",{"id":35},"logging-drivers",[37],{"type":19,"value":38},"Logging drivers",{"type":9,"tag":10,"props":40,"children":41},{},[42],{"type":19,"value":43},"The daemon's logging driver decides where a container's stdout and stderr go.",{"type":9,"tag":45,"props":46,"children":47},"table",{},[48,77],{"type":9,"tag":49,"props":50,"children":51},"thead",{},[52],{"type":9,"tag":53,"props":54,"children":55},"tr",{},[56,62,67],{"type":9,"tag":57,"props":58,"children":59},"th",{},[60],{"type":19,"value":61},"Driver",{"type":9,"tag":57,"props":63,"children":64},{},[65],{"type":19,"value":66},"Where",{"type":9,"tag":57,"props":68,"children":69},{},[70,75],{"type":9,"tag":14,"props":71,"children":73},{"className":72},[],[74],{"type":19,"value":20},{"type":19,"value":76}," works",{"type":9,"tag":78,"props":79,"children":80},"tbody",{},[81,104,125,146,187,215],{"type":9,"tag":53,"props":82,"children":83},{},[84,94,99],{"type":9,"tag":85,"props":86,"children":87},"td",{},[88],{"type":9,"tag":14,"props":89,"children":91},{"className":90},[],[92],{"type":19,"value":93},"json-file",{"type":9,"tag":85,"props":95,"children":96},{},[97],{"type":19,"value":98},"a JSON file per container — the historical default",{"type":9,"tag":85,"props":100,"children":101},{},[102],{"type":19,"value":103},"yes",{"type":9,"tag":53,"props":105,"children":106},{},[107,116,121],{"type":9,"tag":85,"props":108,"children":109},{},[110],{"type":9,"tag":14,"props":111,"children":113},{"className":112},[],[114],{"type":19,"value":115},"local",{"type":9,"tag":85,"props":117,"children":118},{},[119],{"type":19,"value":120},"a compressed, rotated binary format",{"type":9,"tag":85,"props":122,"children":123},{},[124],{"type":19,"value":103},{"type":9,"tag":53,"props":126,"children":127},{},[128,137,142],{"type":9,"tag":85,"props":129,"children":130},{},[131],{"type":9,"tag":14,"props":132,"children":134},{"className":133},[],[135],{"type":19,"value":136},"journald",{"type":9,"tag":85,"props":138,"children":139},{},[140],{"type":19,"value":141},"systemd's journal",{"type":9,"tag":85,"props":143,"children":144},{},[145],{"type":19,"value":103},{"type":9,"tag":53,"props":147,"children":148},{},[149,173,178],{"type":9,"tag":85,"props":150,"children":151},{},[152,158,160,166,167],{"type":9,"tag":14,"props":153,"children":155},{"className":154},[],[156],{"type":19,"value":157},"syslog",{"type":19,"value":159}," / ",{"type":9,"tag":14,"props":161,"children":163},{"className":162},[],[164],{"type":19,"value":165},"fluentd",{"type":19,"value":159},{"type":9,"tag":14,"props":168,"children":170},{"className":169},[],[171],{"type":19,"value":172},"gelf",{"type":9,"tag":85,"props":174,"children":175},{},[176],{"type":19,"value":177},"a remote collector",{"type":9,"tag":85,"props":179,"children":180},{},[181],{"type":9,"tag":182,"props":183,"children":184},"strong",{},[185],{"type":19,"value":186},"no",{"type":9,"tag":53,"props":188,"children":189},{},[190,206,211],{"type":9,"tag":85,"props":191,"children":192},{},[193,199,200],{"type":9,"tag":14,"props":194,"children":196},{"className":195},[],[197],{"type":19,"value":198},"awslogs",{"type":19,"value":159},{"type":9,"tag":14,"props":201,"children":203},{"className":202},[],[204],{"type":19,"value":205},"gcplogs",{"type":9,"tag":85,"props":207,"children":208},{},[209],{"type":19,"value":210},"a cloud provider's log service",{"type":9,"tag":85,"props":212,"children":213},{},[214],{"type":19,"value":186},{"type":9,"tag":53,"props":216,"children":217},{},[218,227,232],{"type":9,"tag":85,"props":219,"children":220},{},[221],{"type":9,"tag":14,"props":222,"children":224},{"className":223},[],[225],{"type":19,"value":226},"none",{"type":9,"tag":85,"props":228,"children":229},{},[230],{"type":19,"value":231},"discarded",{"type":9,"tag":85,"props":233,"children":234},{},[235],{"type":19,"value":186},{"type":9,"tag":237,"props":238,"children":244},"pre",{"className":239,"code":241,"language":242,"meta":243},[240],"language-json","{\n  \"log-driver\": \"local\",\n  \"log-opts\": { \"max-size\": \"10m\", \"max-file\": \"3\", \"compress\": \"true\" }\n}\n","json","",[245],{"type":9,"tag":14,"props":246,"children":247},{"__ignoreMap":243},[248],{"type":19,"value":241},{"type":9,"tag":10,"props":250,"children":251},{},[252,262,264,269],{"type":9,"tag":182,"props":253,"children":254},{},[255,260],{"type":9,"tag":14,"props":256,"children":258},{"className":257},[],[259],{"type":19,"value":115},{"type":19,"value":261}," is the right default on a real host.",{"type":19,"value":263}," It rotates, compresses, and is cheaper to write than ",{"type":9,"tag":14,"props":265,"children":267},{"className":266},[],[268],{"type":19,"value":93},{"type":19,"value":270},". Engine 29.5 added custom attributes to it, so entries can be tagged at the driver level rather than by the application.",{"data":272,"body":273},{},{"type":6,"children":274},[275,280],{"type":9,"tag":276,"props":277,"children":279},"terminal-teaser",{":lines":278},"[{\"cmd\":\"docker info --format '{{.LoggingDriver}}'\",\"out\":\"local\"},{\"cmd\":\"docker inspect -f '{{.HostConfig.LogConfig.Type}}' api\",\"out\":\"local\"},{\"cmd\":\"du -sh /var/lib/docker/containers/*/*-json.log 2>/dev/null | sort -h | tail -3\",\"out\":\"2.1G  .../f3a1...-json.log\\n7.8G  .../9c2e...-json.log\"}]",[],{"type":9,"tag":10,"props":281,"children":282},{},[283,295,297,303],{"type":9,"tag":182,"props":284,"children":285},{},[286,288,293],{"type":19,"value":287},"The default ",{"type":9,"tag":14,"props":289,"children":291},{"className":290},[],[292],{"type":19,"value":93},{"type":19,"value":294}," driver has no size limit.",{"type":19,"value":296}," A chatty container will write until the disk is full, and because the files are not volumes they never appear in ",{"type":9,"tag":14,"props":298,"children":300},{"className":299},[],[301],{"type":19,"value":302},"docker system df",{"type":19,"value":304},". That second command is the one to run on any host you have inherited.",{"data":306,"body":307},{},{"type":6,"children":308},[309],{"type":9,"tag":310,"props":311,"children":316},"quiz",{":answer":312,":options":313,"explanation":314,"question":315},"0","[\"The default `json-file` log driver has no rotation, so container logs grow without limit\",\"Stopped containers are never deleted\",\"The build cache is never pruned\"]","All three are real, and the logs are the one that is invisible — they live under `/var/lib/docker/containers/`, not in anything `docker system df` reports. Set `max-size` and `max-file` globally in `daemon.json`, not per container, so nothing can be started without them.","Why does an unconfigured Docker host eventually run out of disk even with no large images or volumes?",[],{"data":318,"body":319},{},{"type":6,"children":320},[321,332,358,383],{"type":9,"tag":33,"props":322,"children":324},{"id":323},"forwarding-without-giving-up-docker-logs",[325,327],{"type":19,"value":326},"Forwarding without giving up ",{"type":9,"tag":14,"props":328,"children":330},{"className":329},[],[331],{"type":19,"value":20},{"type":9,"tag":10,"props":333,"children":334},{},[335,337,342,344,349,351,356],{"type":19,"value":336},"A remote driver like ",{"type":9,"tag":14,"props":338,"children":340},{"className":339},[],[341],{"type":19,"value":165},{"type":19,"value":343}," or ",{"type":9,"tag":14,"props":345,"children":347},{"className":346},[],[348],{"type":19,"value":172},{"type":19,"value":350}," breaks ",{"type":9,"tag":14,"props":352,"children":354},{"className":353},[],[355],{"type":19,"value":20},{"type":19,"value":357},", which is the command everyone reaches for first at 3am. The usual production shape avoids that trade:",{"type":9,"tag":10,"props":359,"children":360},{},[361,366,368,374,376,381],{"type":9,"tag":182,"props":362,"children":363},{},[364],{"type":19,"value":365},"Write locally with rotation, and ship the files with a collector.",{"type":19,"value":367}," A sidecar or host agent — Fluent Bit, Vector, Promtail, the OpenTelemetry Collector — reads ",{"type":9,"tag":14,"props":369,"children":371},{"className":370},[],[372],{"type":19,"value":373},"/var/lib/docker/containers/*/*.log",{"type":19,"value":375},", enriches with container metadata, and forwards. ",{"type":9,"tag":14,"props":377,"children":379},{"className":378},[],[380],{"type":19,"value":20},{"type":19,"value":382}," keeps working; the logs also reach your aggregator.",{"type":9,"tag":10,"props":384,"children":385},{},[386,388,393],{"type":19,"value":387},"Then, in the application: ",{"type":9,"tag":182,"props":389,"children":390},{},[391],{"type":19,"value":392},"log JSON to stdout and nothing else.",{"type":19,"value":394}," No files, no log rotation inside the container, no syslog. One stream, structured, and the platform decides where it goes. A container writing its own log files is producing data nobody collects and disk nobody bounded.",{"data":396,"body":397},{},{"type":6,"children":398},[399,405,416],{"type":9,"tag":33,"props":400,"children":402},{"id":401},"events",[403],{"type":19,"value":404},"Events",{"type":9,"tag":10,"props":406,"children":407},{},[408,414],{"type":9,"tag":14,"props":409,"children":411},{"className":410},[],[412],{"type":19,"value":413},"docker events",{"type":19,"value":415}," is the daemon's activity stream, and it answers questions logs cannot:",{"type":9,"tag":237,"props":417,"children":421},{"className":418,"code":420,"language":19},[419],"language-text","docker events --since 1h --filter event=oom\ndocker events --filter container=api --filter event=restart\ndocker events --filter type=image --filter event=pull\ndocker events --format '{{.Time}} {{.Type}} {{.Action}} {{.Actor.Attributes.name}}'\n",[422],{"type":9,"tag":14,"props":423,"children":424},{"__ignoreMap":243},[425],{"type":19,"value":420},{"data":427,"body":428},{},{"type":6,"children":429},[430,434,470],{"type":9,"tag":276,"props":431,"children":433},{":lines":432},"[{\"cmd\":\"docker events --since 30m --filter event=die --format '{{.Time}} {{.Actor.Attributes.name}} exit={{.Actor.Attributes.exitCode}}'\",\"out\":\"1756370412 api exit=137\\n1756370498 api exit=137\\n1756370671 api exit=137\"}]",[],{"type":9,"tag":10,"props":435,"children":436},{},[437,439,445,447,453,455,460,462,468],{"type":19,"value":438},"Three OOM kills in four minutes, with timestamps. ",{"type":9,"tag":14,"props":440,"children":442},{"className":441},[],[443],{"type":19,"value":444},"docker ps",{"type":19,"value":446}," would have shown that container as ",{"type":9,"tag":14,"props":448,"children":450},{"className":449},[],[451],{"type":19,"value":452},"Up 12 seconds",{"type":19,"value":454}," and looking healthy — because it is up, for the fourth time. ",{"type":9,"tag":182,"props":456,"children":457},{},[458],{"type":19,"value":459},"Events are how a crash loop stops hiding",{"type":19,"value":461},", and ",{"type":9,"tag":14,"props":463,"children":465},{"className":464},[],[466],{"type":19,"value":467},"RestartCount",{"type":19,"value":469}," is the other half:",{"type":9,"tag":237,"props":471,"children":474},{"className":472,"code":473,"language":19},[419],"docker inspect -f '{{.RestartCount}} {{.State.Status}}' api\n",[475],{"type":9,"tag":14,"props":476,"children":477},{"__ignoreMap":243},[478],{"type":19,"value":473},{"data":480,"body":481},{},{"type":6,"children":482},[483],{"type":9,"tag":310,"props":484,"children":488},{":answer":312,":options":485,"explanation":486,"question":487},"[\"`docker events` and `RestartCount` — it is restarting repeatedly and each `Up` is a fresh start\",\"`docker logs`, which would show the errors\",\"`docker stats`, which shows CPU spikes\"]","`docker ps` shows uptime of the *current* attempt, so a crash loop with backoff looks like a young, healthy container. `RestartCount` in the hundreds against an 8-second uptime is the giveaway; `docker events --filter event=die` gives you the exit codes and timing.","A container shows `Up 8 seconds` in `docker ps` and appears healthy, but users report intermittent errors. What reveals the problem?",[],{"data":490,"body":491},{},{"type":6,"children":492},[493,499,508,513,576],{"type":9,"tag":33,"props":494,"children":496},{"id":495},"metrics",[497],{"type":19,"value":498},"Metrics",{"type":9,"tag":237,"props":500,"children":503},{"className":501,"code":502,"language":19},[419],"docker stats\ndocker stats --no-stream --format '{{.Name}}\\t{{.CPUPerc}}\\t{{.MemUsage}}\\t{{.PIDs}}'\n",[504],{"type":9,"tag":14,"props":505,"children":506},{"__ignoreMap":243},[507],{"type":19,"value":502},{"type":9,"tag":10,"props":509,"children":510},{},[511],{"type":19,"value":512},"Fine interactively, useless as a record — it does not persist. For real metrics, read cgroups directly or run an exporter:",{"type":9,"tag":514,"props":515,"children":516},"ul",{},[517,528,562],{"type":9,"tag":518,"props":519,"children":520},"li",{},[521,526],{"type":9,"tag":182,"props":522,"children":523},{},[524],{"type":19,"value":525},"cAdvisor",{"type":19,"value":527}," exposes per-container CPU, memory, network and disk in Prometheus format.",{"type":9,"tag":518,"props":529,"children":530},{},[531,536,538,544,546,552,554,560],{"type":9,"tag":182,"props":532,"children":533},{},[534],{"type":19,"value":535},"The daemon itself",{"type":19,"value":537}," can expose Prometheus metrics via ",{"type":9,"tag":14,"props":539,"children":541},{"className":540},[],[542],{"type":19,"value":543},"\"metrics-addr\"",{"type":19,"value":545}," in ",{"type":9,"tag":14,"props":547,"children":549},{"className":548},[],[550],{"type":19,"value":551},"daemon.json",{"type":19,"value":553}," — those are about ",{"type":9,"tag":14,"props":555,"children":557},{"className":556},[],[558],{"type":19,"value":559},"dockerd",{"type":19,"value":561},", not your containers.",{"type":9,"tag":518,"props":563,"children":564},{},[565,574],{"type":9,"tag":182,"props":566,"children":567},{},[568],{"type":9,"tag":14,"props":569,"children":571},{"className":570},[],[572],{"type":19,"value":573},"/sys/fs/cgroup/.../memory.events",{"type":19,"value":575}," is the ground truth for OOM kills and reclaim pressure, as covered in the limits lesson.",{"type":9,"tag":10,"props":577,"children":578},{},[579,581,586,588,593,594,599,601,606,608,614],{"type":19,"value":580},"The container-specific metrics worth alerting on, as opposed to the generic host ones: ",{"type":9,"tag":182,"props":582,"children":583},{},[584],{"type":19,"value":585},"restart count increasing",{"type":19,"value":587},", ",{"type":9,"tag":182,"props":589,"children":590},{},[591],{"type":19,"value":592},"OOM kill events",{"type":19,"value":587},{"type":9,"tag":182,"props":595,"children":596},{},[597],{"type":19,"value":598},"CPU throttling time",{"type":19,"value":600}," (a throttled container is slow, not busy, and CPU-percent alerts miss it entirely), and ",{"type":9,"tag":182,"props":602,"children":603},{},[604],{"type":19,"value":605},"health status transitions",{"type":19,"value":607},". Engine 29 added ",{"type":9,"tag":14,"props":609,"children":611},{"className":610},[],[612],{"type":19,"value":613},"Health",{"type":19,"value":615}," to the container list API, so a collector can read health without inspecting each container.",{"data":617,"body":618},{},{"type":6,"children":619},[620],{"type":9,"tag":621,"props":622,"children":627},"fill-blank",{":answer":623,"hint":624,"placeholder":625,"prompt":626},"[\"docker events --since 1h --filter event=oom\",\"docker events --filter event=oom --since 1h\",\"docker events --since 1h --filter event=oom --filter type=container\"]","The events command, a time window, and a filter on the event name.","docker events ...","Show container OOM-kill events from the last hour.",[],{"data":629,"body":630},{},{"type":6,"children":631},[632,795],{"type":9,"tag":633,"props":634,"children":636},"deep-dive",{"title":635},"Correlating a container back to a host process",[637,642,650,659,669,678,683,692,702,711,762],{"type":9,"tag":10,"props":638,"children":639},{},[640],{"type":19,"value":641},"When something is wrong at the host level — CPU pinned, disk saturated, a suspicious network connection — you have a PID and need to know which container it belongs to. Or the reverse.",{"type":9,"tag":10,"props":643,"children":644},{},[645],{"type":9,"tag":182,"props":646,"children":647},{},[648],{"type":19,"value":649},"Container to host PID:",{"type":9,"tag":237,"props":651,"children":654},{"className":652,"code":653,"language":19},[419],"docker inspect -f '{{.State.Pid}}' api\n",[655],{"type":9,"tag":14,"props":656,"children":657},{"__ignoreMap":243},[658],{"type":19,"value":653},{"type":9,"tag":10,"props":660,"children":661},{},[662,667],{"type":9,"tag":182,"props":663,"children":664},{},[665],{"type":19,"value":666},"Host PID to container",{"type":19,"value":668},", which is the direction you usually need:",{"type":9,"tag":237,"props":670,"children":673},{"className":671,"code":672,"language":19},[419],"cat /proc/48213/cgroup\n0::/system.slice/docker-7c1f9a3e4d82ab....scope\n",[674],{"type":9,"tag":14,"props":675,"children":676},{"__ignoreMap":243},[677],{"type":19,"value":672},{"type":9,"tag":10,"props":679,"children":680},{},[681],{"type":19,"value":682},"The cgroup path contains the container ID. From there:",{"type":9,"tag":237,"props":684,"children":687},{"className":685,"code":686,"language":19},[419],"docker inspect --format '{{.Name}}' 7c1f9a3e4d82\n",[688],{"type":9,"tag":14,"props":689,"children":690},{"__ignoreMap":243},[691],{"type":19,"value":686},{"type":9,"tag":10,"props":693,"children":694},{},[695,700],{"type":9,"tag":182,"props":696,"children":697},{},[698],{"type":19,"value":699},"Everything a container is doing, from the host",{"type":19,"value":701},", without a shell inside it:",{"type":9,"tag":237,"props":703,"children":706},{"className":704,"code":705,"language":19},[419],"docker top api                                    # its processes\nsudo nsenter -t 48213 -n ss -tulpn                # its sockets, host tooling\nsudo ls -l /proc/48213/fd | wc -l                 # open file descriptors\n",[707],{"type":9,"tag":14,"props":708,"children":709},{"__ignoreMap":243},[710],{"type":19,"value":705},{"type":9,"tag":10,"props":712,"children":713},{},[714,716,722,724,729,731,736,738,744,746,752,754,760],{"type":19,"value":715},"That ",{"type":9,"tag":14,"props":717,"children":719},{"className":718},[],[720],{"type":19,"value":721},"nsenter",{"type":19,"value":723}," is the technique worth remembering. It runs a ",{"type":9,"tag":182,"props":725,"children":726},{},[727],{"type":19,"value":728},"host binary",{"type":19,"value":730}," inside the ",{"type":9,"tag":182,"props":732,"children":733},{},[734],{"type":19,"value":735},"container's namespaces",{"type":19,"value":737}," — so you can inspect the network of a distroless container that has no shell, no ",{"type":9,"tag":14,"props":739,"children":741},{"className":740},[],[742],{"type":19,"value":743},"ss",{"type":19,"value":745},", and no ",{"type":9,"tag":14,"props":747,"children":749},{"className":748},[],[750],{"type":19,"value":751},"netstat",{"type":19,"value":753},". It is ",{"type":9,"tag":14,"props":755,"children":757},{"className":756},[],[758],{"type":19,"value":759},"docker exec",{"type":19,"value":761}," without the requirement that the tool exist in the image, and it is the reason minimal images are debuggable at all.",{"type":9,"tag":10,"props":763,"children":764},{},[765,767,772,774,780,781,787,788,793],{"type":19,"value":766},"The general principle from the first lesson holds throughout: ",{"type":9,"tag":182,"props":768,"children":769},{},[770],{"type":19,"value":771},"a container is a host process with namespaces and a cgroup attached.",{"type":19,"value":773}," When the Docker-shaped tooling runs out, ",{"type":9,"tag":14,"props":775,"children":777},{"className":776},[],[778],{"type":19,"value":779},"/proc",{"type":19,"value":587},{"type":9,"tag":14,"props":782,"children":784},{"className":783},[],[785],{"type":19,"value":786},"/sys/fs/cgroup",{"type":19,"value":461},{"type":9,"tag":14,"props":789,"children":791},{"className":790},[],[792],{"type":19,"value":721},{"type":19,"value":794}," still answer the question.",{"type":9,"tag":10,"props":796,"children":797},{},[798],{"type":19,"value":799},"Next up: the last lesson — running AI workloads, with Model Runner and Compose's model support.",1787908868462]