[{"data":1,"prerenderedAt":512},["ShallowReactive",2],{"en-articles":3},[4],{"id":5,"title":6,"body":7,"category":496,"coverImage":497,"date":498,"description":499,"draft":500,"extension":501,"featured":426,"meta":502,"navigation":426,"path":503,"readingTime":423,"seo":504,"stem":505,"tags":506,"updatedAt":497,"__hash__":511},"articles\u002Fen\u002Farticles\u002Fmysql-job-queue.md","A Reliable Job Queue on MySQL Without Redis",{"type":8,"value":9,"toc":489},"minimark",[10,14,17,22,34,37,60,67,169,174,178,468,478,482,485],[11,12,13],"p",{},"When most developers hear \"job queue,\" they immediately think Redis. That's reasonable — Redis is fast, simple, and Laravel supports it out of the box. But Redis has a cost: it's another service in the infrastructure. Another failure point. Another thing to monitor.",[11,15,16],{},"Three years ago we faced a choice: stand up Redis, or build a queue on MySQL we already had. We chose MySQL. No regrets.",[18,19,21],"h2",{"id":20},"why-mysql","Why MySQL?",[11,23,24,25,29,30,33],{},"The \"Redis is faster\" argument is true but incomplete. It's faster for in-memory operations. But when a job runs for seconds or minutes, the difference between ",[26,27,28],"code",{},"RPOP"," and ",[26,31,32],{},"SELECT ... FOR UPDATE SKIP LOCKED"," is statistically irrelevant.",[11,35,36],{},"MySQL gives us:",[38,39,40,48,54],"ul",{},[41,42,43,47],"li",{},[44,45,46],"strong",{},"ACID guarantees"," — a job won't disappear even if everything crashes mid-execution",[41,49,50,53],{},[44,51,52],{},"Transactions"," — create a job and update business data in a single transaction",[41,55,56,59],{},[44,57,58],{},"Visibility"," — everything is visible in standard MySQL tooling",[18,61,63,64],{"id":62},"the-key-pattern-skip-locked","The key pattern: ",[26,65,66],{},"SKIP LOCKED",[68,69,74],"pre",{"className":70,"code":71,"language":72,"meta":73,"style":73},"language-sql shiki shiki-themes github-light github-dark","SELECT * FROM jobs\nWHERE status = 'pending'\n  AND available_at \u003C= NOW()\nORDER BY priority DESC, id ASC\nLIMIT 1\nFOR UPDATE SKIP LOCKED;\n","sql","",[26,75,76,95,111,129,144,154],{"__ignoreMap":73},[77,78,81,85,88,91],"span",{"class":79,"line":80},"line",1,[77,82,84],{"class":83},"szBVR","SELECT",[77,86,87],{"class":83}," *",[77,89,90],{"class":83}," FROM",[77,92,94],{"class":93},"sVt8B"," jobs\n",[77,96,98,101,104,107],{"class":79,"line":97},2,[77,99,100],{"class":83},"WHERE",[77,102,103],{"class":83}," status",[77,105,106],{"class":83}," =",[77,108,110],{"class":109},"sZZnC"," 'pending'\n",[77,112,114,117,120,123,126],{"class":79,"line":113},3,[77,115,116],{"class":83},"  AND",[77,118,119],{"class":93}," available_at ",[77,121,122],{"class":83},"\u003C=",[77,124,125],{"class":83}," NOW",[77,127,128],{"class":93},"()\n",[77,130,132,135,138,141],{"class":79,"line":131},4,[77,133,134],{"class":83},"ORDER BY",[77,136,137],{"class":83}," priority DESC",[77,139,140],{"class":93},", id ",[77,142,143],{"class":83},"ASC\n",[77,145,147,150],{"class":79,"line":146},5,[77,148,149],{"class":83},"LIMIT",[77,151,153],{"class":152},"sj4cs"," 1\n",[77,155,157,160,163,166],{"class":79,"line":156},6,[77,158,159],{"class":83},"FOR",[77,161,162],{"class":83}," UPDATE",[77,164,165],{"class":83}," SKIP",[77,167,168],{"class":93}," LOCKED;\n",[11,170,171,173],{},[26,172,66],{}," tells MySQL: \"Give me the first row nobody else has locked.\" Without it, workers would compete for the same jobs.",[18,175,177],{"id":176},"table-design","Table design",[68,179,181],{"className":70,"code":180,"language":72,"meta":73,"style":73},"CREATE TABLE jobs (\n  id           BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,\n  queue        VARCHAR(100) NOT NULL DEFAULT 'default',\n  payload      JSON NOT NULL,\n  status       ENUM('pending', 'processing', 'done', 'failed') NOT NULL DEFAULT 'pending',\n  attempts     TINYINT UNSIGNED NOT NULL DEFAULT 0,\n  priority     TINYINT UNSIGNED NOT NULL DEFAULT 5,\n  available_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,\n  locked_at    DATETIME NULL,\n  locked_by    VARCHAR(100) NULL,\n  created_at   DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,\n  failed_at    DATETIME NULL,\n  error        TEXT NULL,\n\n  INDEX idx_queue_status_available (queue, status, available_at),\n  INDEX idx_locked_by (locked_by)\n) ENGINE=InnoDB;\n",[26,182,183,198,215,243,256,294,314,333,349,362,382,396,408,421,428,448,456],{"__ignoreMap":73},[77,184,185,188,191,195],{"class":79,"line":80},[77,186,187],{"class":83},"CREATE",[77,189,190],{"class":83}," TABLE",[77,192,194],{"class":193},"sScJk"," jobs",[77,196,197],{"class":93}," (\n",[77,199,200,203,206,209,212],{"class":79,"line":97},[77,201,202],{"class":93},"  id           ",[77,204,205],{"class":83},"BIGINT",[77,207,208],{"class":93}," UNSIGNED AUTO_INCREMENT ",[77,210,211],{"class":83},"PRIMARY KEY",[77,213,214],{"class":93},",\n",[77,216,217,220,223,226,229,232,235,238,241],{"class":79,"line":113},[77,218,219],{"class":83},"  queue",[77,221,222],{"class":83},"        VARCHAR",[77,224,225],{"class":93},"(",[77,227,228],{"class":152},"100",[77,230,231],{"class":93},") ",[77,233,234],{"class":83},"NOT NULL",[77,236,237],{"class":83}," DEFAULT",[77,239,240],{"class":109}," 'default'",[77,242,214],{"class":93},[77,244,245,248,251,254],{"class":79,"line":131},[77,246,247],{"class":93},"  payload      ",[77,249,250],{"class":83},"JSON",[77,252,253],{"class":83}," NOT NULL",[77,255,214],{"class":93},[77,257,258,261,264,267,270,273,275,278,280,283,285,287,289,292],{"class":79,"line":146},[77,259,260],{"class":83},"  status",[77,262,263],{"class":93},"       ENUM(",[77,265,266],{"class":109},"'pending'",[77,268,269],{"class":93},", ",[77,271,272],{"class":109},"'processing'",[77,274,269],{"class":93},[77,276,277],{"class":109},"'done'",[77,279,269],{"class":93},[77,281,282],{"class":109},"'failed'",[77,284,231],{"class":93},[77,286,234],{"class":83},[77,288,237],{"class":83},[77,290,291],{"class":109}," 'pending'",[77,293,214],{"class":93},[77,295,296,299,302,305,307,309,312],{"class":79,"line":156},[77,297,298],{"class":93},"  attempts     ",[77,300,301],{"class":83},"TINYINT",[77,303,304],{"class":93}," UNSIGNED ",[77,306,234],{"class":83},[77,308,237],{"class":83},[77,310,311],{"class":152}," 0",[77,313,214],{"class":93},[77,315,317,320,322,324,326,328,331],{"class":79,"line":316},7,[77,318,319],{"class":93},"  priority     ",[77,321,301],{"class":83},[77,323,304],{"class":93},[77,325,234],{"class":83},[77,327,237],{"class":83},[77,329,330],{"class":152}," 5",[77,332,214],{"class":93},[77,334,336,339,342,344,346],{"class":79,"line":335},8,[77,337,338],{"class":93},"  available_at ",[77,340,341],{"class":83},"DATETIME",[77,343,253],{"class":83},[77,345,237],{"class":83},[77,347,348],{"class":93}," CURRENT_TIMESTAMP,\n",[77,350,352,355,357,360],{"class":79,"line":351},9,[77,353,354],{"class":93},"  locked_at    ",[77,356,341],{"class":83},[77,358,359],{"class":83}," NULL",[77,361,214],{"class":93},[77,363,365,368,371,373,375,377,380],{"class":79,"line":364},10,[77,366,367],{"class":93},"  locked_by    ",[77,369,370],{"class":83},"VARCHAR",[77,372,225],{"class":93},[77,374,228],{"class":152},[77,376,231],{"class":93},[77,378,379],{"class":83},"NULL",[77,381,214],{"class":93},[77,383,385,388,390,392,394],{"class":79,"line":384},11,[77,386,387],{"class":93},"  created_at   ",[77,389,341],{"class":83},[77,391,253],{"class":83},[77,393,237],{"class":83},[77,395,348],{"class":93},[77,397,399,402,404,406],{"class":79,"line":398},12,[77,400,401],{"class":93},"  failed_at    ",[77,403,341],{"class":83},[77,405,359],{"class":83},[77,407,214],{"class":93},[77,409,411,414,417,419],{"class":79,"line":410},13,[77,412,413],{"class":93},"  error        ",[77,415,416],{"class":83},"TEXT",[77,418,359],{"class":83},[77,420,214],{"class":93},[77,422,424],{"class":79,"line":423},14,[77,425,427],{"emptyLinePlaceholder":426},true,"\n",[77,429,431,434,437,440,442,445],{"class":79,"line":430},15,[77,432,433],{"class":83},"  INDEX",[77,435,436],{"class":93}," idx_queue_status_available (",[77,438,439],{"class":83},"queue",[77,441,269],{"class":93},[77,443,444],{"class":83},"status",[77,446,447],{"class":93},", available_at),\n",[77,449,451,453],{"class":79,"line":450},16,[77,452,433],{"class":83},[77,454,455],{"class":93}," idx_locked_by (locked_by)\n",[77,457,459,462,465],{"class":79,"line":458},17,[77,460,461],{"class":93},") ENGINE",[77,463,464],{"class":83},"=",[77,466,467],{"class":93},"InnoDB;\n",[11,469,470,471,474,475,477],{},"The index on ",[26,472,473],{},"(queue, status, available_at)"," is where all the performance lives. Without it, ",[26,476,32],{}," scans the whole table.",[18,479,481],{"id":480},"conclusions","Conclusions",[11,483,484],{},"Three years, millions of jobs — zero lost jobs, zero duplicates. MySQL held up.",[486,487,488],"style",{},"html pre.shiki code .szBVR, html code.shiki .szBVR{--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}",{"title":73,"searchDepth":97,"depth":97,"links":490},[491,492,494,495],{"id":20,"depth":97,"text":21},{"id":62,"depth":97,"text":493},"The key pattern: SKIP LOCKED",{"id":176,"depth":97,"text":177},{"id":480,"depth":97,"text":481},"Backend",null,"2024-08-15","How we built a production-ready job queue on MySQL that survived 3 years of load without a single failure.",false,"md",{},"\u002Fen\u002Farticles\u002Fmysql-job-queue",{"title":6,"description":499},"en\u002Farticles\u002Fmysql-job-queue",[507,508,509,496,510],"MySQL","Queues","Laravel","Performance","8IOQmiTkXybYNMyVfGv8SeOk6L0Tkpjl_GegkgHhLJU",1783366806767]