sequencer.c 107 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838
  1. /*
  2. * Copyright Altera Corporation (C) 2012-2015
  3. *
  4. * SPDX-License-Identifier: BSD-3-Clause
  5. */
  6. #include <common.h>
  7. #include <asm/io.h>
  8. #include <asm/arch/sdram.h>
  9. #include <errno.h>
  10. #include "sequencer.h"
  11. #include "sequencer_auto.h"
  12. #include "sequencer_auto_ac_init.h"
  13. #include "sequencer_auto_inst_init.h"
  14. #include "sequencer_defines.h"
  15. static struct socfpga_sdr_rw_load_manager *sdr_rw_load_mgr_regs =
  16. (struct socfpga_sdr_rw_load_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0x800);
  17. static struct socfpga_sdr_rw_load_jump_manager *sdr_rw_load_jump_mgr_regs =
  18. (struct socfpga_sdr_rw_load_jump_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0xC00);
  19. static struct socfpga_sdr_reg_file *sdr_reg_file =
  20. (struct socfpga_sdr_reg_file *)SDR_PHYGRP_REGFILEGRP_ADDRESS;
  21. static struct socfpga_sdr_scc_mgr *sdr_scc_mgr =
  22. (struct socfpga_sdr_scc_mgr *)(SDR_PHYGRP_SCCGRP_ADDRESS | 0xe00);
  23. static struct socfpga_phy_mgr_cmd *phy_mgr_cmd =
  24. (struct socfpga_phy_mgr_cmd *)SDR_PHYGRP_PHYMGRGRP_ADDRESS;
  25. static struct socfpga_phy_mgr_cfg *phy_mgr_cfg =
  26. (struct socfpga_phy_mgr_cfg *)(SDR_PHYGRP_PHYMGRGRP_ADDRESS | 0x40);
  27. static struct socfpga_data_mgr *data_mgr =
  28. (struct socfpga_data_mgr *)SDR_PHYGRP_DATAMGRGRP_ADDRESS;
  29. static struct socfpga_sdr_ctrl *sdr_ctrl =
  30. (struct socfpga_sdr_ctrl *)SDR_CTRLGRP_ADDRESS;
  31. #define DELTA_D 1
  32. /*
  33. * In order to reduce ROM size, most of the selectable calibration steps are
  34. * decided at compile time based on the user's calibration mode selection,
  35. * as captured by the STATIC_CALIB_STEPS selection below.
  36. *
  37. * However, to support simulation-time selection of fast simulation mode, where
  38. * we skip everything except the bare minimum, we need a few of the steps to
  39. * be dynamic. In those cases, we either use the DYNAMIC_CALIB_STEPS for the
  40. * check, which is based on the rtl-supplied value, or we dynamically compute
  41. * the value to use based on the dynamically-chosen calibration mode
  42. */
  43. #define DLEVEL 0
  44. #define STATIC_IN_RTL_SIM 0
  45. #define STATIC_SKIP_DELAY_LOOPS 0
  46. #define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | \
  47. STATIC_SKIP_DELAY_LOOPS)
  48. /* calibration steps requested by the rtl */
  49. uint16_t dyn_calib_steps;
  50. /*
  51. * To make CALIB_SKIP_DELAY_LOOPS a dynamic conditional option
  52. * instead of static, we use boolean logic to select between
  53. * non-skip and skip values
  54. *
  55. * The mask is set to include all bits when not-skipping, but is
  56. * zero when skipping
  57. */
  58. uint16_t skip_delay_mask; /* mask off bits when skipping/not-skipping */
  59. #define SKIP_DELAY_LOOP_VALUE_OR_ZERO(non_skip_value) \
  60. ((non_skip_value) & skip_delay_mask)
  61. struct gbl_type *gbl;
  62. struct param_type *param;
  63. uint32_t curr_shadow_reg;
  64. static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
  65. uint32_t write_group, uint32_t use_dm,
  66. uint32_t all_correct, uint32_t *bit_chk, uint32_t all_ranks);
  67. static void set_failing_group_stage(uint32_t group, uint32_t stage,
  68. uint32_t substage)
  69. {
  70. /*
  71. * Only set the global stage if there was not been any other
  72. * failing group
  73. */
  74. if (gbl->error_stage == CAL_STAGE_NIL) {
  75. gbl->error_substage = substage;
  76. gbl->error_stage = stage;
  77. gbl->error_group = group;
  78. }
  79. }
  80. static void reg_file_set_group(u16 set_group)
  81. {
  82. clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff0000, set_group << 16);
  83. }
  84. static void reg_file_set_stage(u8 set_stage)
  85. {
  86. clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff, set_stage & 0xff);
  87. }
  88. static void reg_file_set_sub_stage(u8 set_sub_stage)
  89. {
  90. set_sub_stage &= 0xff;
  91. clrsetbits_le32(&sdr_reg_file->cur_stage, 0xff00, set_sub_stage << 8);
  92. }
  93. /**
  94. * phy_mgr_initialize() - Initialize PHY Manager
  95. *
  96. * Initialize PHY Manager.
  97. */
  98. static void phy_mgr_initialize(void)
  99. {
  100. u32 ratio;
  101. debug("%s:%d\n", __func__, __LINE__);
  102. /* Calibration has control over path to memory */
  103. /*
  104. * In Hard PHY this is a 2-bit control:
  105. * 0: AFI Mux Select
  106. * 1: DDIO Mux Select
  107. */
  108. writel(0x3, &phy_mgr_cfg->mux_sel);
  109. /* USER memory clock is not stable we begin initialization */
  110. writel(0, &phy_mgr_cfg->reset_mem_stbl);
  111. /* USER calibration status all set to zero */
  112. writel(0, &phy_mgr_cfg->cal_status);
  113. writel(0, &phy_mgr_cfg->cal_debug_info);
  114. /* Init params only if we do NOT skip calibration. */
  115. if ((dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL)
  116. return;
  117. ratio = RW_MGR_MEM_DQ_PER_READ_DQS /
  118. RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS;
  119. param->read_correct_mask_vg = (1 << ratio) - 1;
  120. param->write_correct_mask_vg = (1 << ratio) - 1;
  121. param->read_correct_mask = (1 << RW_MGR_MEM_DQ_PER_READ_DQS) - 1;
  122. param->write_correct_mask = (1 << RW_MGR_MEM_DQ_PER_WRITE_DQS) - 1;
  123. ratio = RW_MGR_MEM_DATA_WIDTH /
  124. RW_MGR_MEM_DATA_MASK_WIDTH;
  125. param->dm_correct_mask = (1 << ratio) - 1;
  126. }
  127. /**
  128. * set_rank_and_odt_mask() - Set Rank and ODT mask
  129. * @rank: Rank mask
  130. * @odt_mode: ODT mode, OFF or READ_WRITE
  131. *
  132. * Set Rank and ODT mask (On-Die Termination).
  133. */
  134. static void set_rank_and_odt_mask(const u32 rank, const u32 odt_mode)
  135. {
  136. u32 odt_mask_0 = 0;
  137. u32 odt_mask_1 = 0;
  138. u32 cs_and_odt_mask;
  139. if (odt_mode == RW_MGR_ODT_MODE_OFF) {
  140. odt_mask_0 = 0x0;
  141. odt_mask_1 = 0x0;
  142. } else { /* RW_MGR_ODT_MODE_READ_WRITE */
  143. switch (RW_MGR_MEM_NUMBER_OF_RANKS) {
  144. case 1: /* 1 Rank */
  145. /* Read: ODT = 0 ; Write: ODT = 1 */
  146. odt_mask_0 = 0x0;
  147. odt_mask_1 = 0x1;
  148. break;
  149. case 2: /* 2 Ranks */
  150. if (RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 1) {
  151. /*
  152. * - Dual-Slot , Single-Rank (1 CS per DIMM)
  153. * OR
  154. * - RDIMM, 4 total CS (2 CS per DIMM, 2 DIMM)
  155. *
  156. * Since MEM_NUMBER_OF_RANKS is 2, they
  157. * are both single rank with 2 CS each
  158. * (special for RDIMM).
  159. *
  160. * Read: Turn on ODT on the opposite rank
  161. * Write: Turn on ODT on all ranks
  162. */
  163. odt_mask_0 = 0x3 & ~(1 << rank);
  164. odt_mask_1 = 0x3;
  165. } else {
  166. /*
  167. * - Single-Slot , Dual-Rank (2 CS per DIMM)
  168. *
  169. * Read: Turn on ODT off on all ranks
  170. * Write: Turn on ODT on active rank
  171. */
  172. odt_mask_0 = 0x0;
  173. odt_mask_1 = 0x3 & (1 << rank);
  174. }
  175. break;
  176. case 4: /* 4 Ranks */
  177. /* Read:
  178. * ----------+-----------------------+
  179. * | ODT |
  180. * Read From +-----------------------+
  181. * Rank | 3 | 2 | 1 | 0 |
  182. * ----------+-----+-----+-----+-----+
  183. * 0 | 0 | 1 | 0 | 0 |
  184. * 1 | 1 | 0 | 0 | 0 |
  185. * 2 | 0 | 0 | 0 | 1 |
  186. * 3 | 0 | 0 | 1 | 0 |
  187. * ----------+-----+-----+-----+-----+
  188. *
  189. * Write:
  190. * ----------+-----------------------+
  191. * | ODT |
  192. * Write To +-----------------------+
  193. * Rank | 3 | 2 | 1 | 0 |
  194. * ----------+-----+-----+-----+-----+
  195. * 0 | 0 | 1 | 0 | 1 |
  196. * 1 | 1 | 0 | 1 | 0 |
  197. * 2 | 0 | 1 | 0 | 1 |
  198. * 3 | 1 | 0 | 1 | 0 |
  199. * ----------+-----+-----+-----+-----+
  200. */
  201. switch (rank) {
  202. case 0:
  203. odt_mask_0 = 0x4;
  204. odt_mask_1 = 0x5;
  205. break;
  206. case 1:
  207. odt_mask_0 = 0x8;
  208. odt_mask_1 = 0xA;
  209. break;
  210. case 2:
  211. odt_mask_0 = 0x1;
  212. odt_mask_1 = 0x5;
  213. break;
  214. case 3:
  215. odt_mask_0 = 0x2;
  216. odt_mask_1 = 0xA;
  217. break;
  218. }
  219. break;
  220. }
  221. }
  222. cs_and_odt_mask = (0xFF & ~(1 << rank)) |
  223. ((0xFF & odt_mask_0) << 8) |
  224. ((0xFF & odt_mask_1) << 16);
  225. writel(cs_and_odt_mask, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  226. RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
  227. }
  228. /**
  229. * scc_mgr_set() - Set SCC Manager register
  230. * @off: Base offset in SCC Manager space
  231. * @grp: Read/Write group
  232. * @val: Value to be set
  233. *
  234. * This function sets the SCC Manager (Scan Chain Control Manager) register.
  235. */
  236. static void scc_mgr_set(u32 off, u32 grp, u32 val)
  237. {
  238. writel(val, SDR_PHYGRP_SCCGRP_ADDRESS | off | (grp << 2));
  239. }
  240. /**
  241. * scc_mgr_initialize() - Initialize SCC Manager registers
  242. *
  243. * Initialize SCC Manager registers.
  244. */
  245. static void scc_mgr_initialize(void)
  246. {
  247. /*
  248. * Clear register file for HPS. 16 (2^4) is the size of the
  249. * full register file in the scc mgr:
  250. * RFILE_DEPTH = 1 + log2(MEM_DQ_PER_DQS + 1 + MEM_DM_PER_DQS +
  251. * MEM_IF_READ_DQS_WIDTH - 1);
  252. */
  253. int i;
  254. for (i = 0; i < 16; i++) {
  255. debug_cond(DLEVEL == 1, "%s:%d: Clearing SCC RFILE index %u\n",
  256. __func__, __LINE__, i);
  257. scc_mgr_set(SCC_MGR_HHP_RFILE_OFFSET, 0, i);
  258. }
  259. }
  260. static void scc_mgr_set_dqdqs_output_phase(uint32_t write_group, uint32_t phase)
  261. {
  262. scc_mgr_set(SCC_MGR_DQDQS_OUT_PHASE_OFFSET, write_group, phase);
  263. }
  264. static void scc_mgr_set_dqs_bus_in_delay(uint32_t read_group, uint32_t delay)
  265. {
  266. scc_mgr_set(SCC_MGR_DQS_IN_DELAY_OFFSET, read_group, delay);
  267. }
  268. static void scc_mgr_set_dqs_en_phase(uint32_t read_group, uint32_t phase)
  269. {
  270. scc_mgr_set(SCC_MGR_DQS_EN_PHASE_OFFSET, read_group, phase);
  271. }
  272. static void scc_mgr_set_dqs_en_delay(uint32_t read_group, uint32_t delay)
  273. {
  274. scc_mgr_set(SCC_MGR_DQS_EN_DELAY_OFFSET, read_group, delay);
  275. }
  276. static void scc_mgr_set_dqs_io_in_delay(uint32_t delay)
  277. {
  278. scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, RW_MGR_MEM_DQ_PER_WRITE_DQS,
  279. delay);
  280. }
  281. static void scc_mgr_set_dq_in_delay(uint32_t dq_in_group, uint32_t delay)
  282. {
  283. scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, dq_in_group, delay);
  284. }
  285. static void scc_mgr_set_dq_out1_delay(uint32_t dq_in_group, uint32_t delay)
  286. {
  287. scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET, dq_in_group, delay);
  288. }
  289. static void scc_mgr_set_dqs_out1_delay(uint32_t delay)
  290. {
  291. scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET, RW_MGR_MEM_DQ_PER_WRITE_DQS,
  292. delay);
  293. }
  294. static void scc_mgr_set_dm_out1_delay(uint32_t dm, uint32_t delay)
  295. {
  296. scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET,
  297. RW_MGR_MEM_DQ_PER_WRITE_DQS + 1 + dm,
  298. delay);
  299. }
  300. /* load up dqs config settings */
  301. static void scc_mgr_load_dqs(uint32_t dqs)
  302. {
  303. writel(dqs, &sdr_scc_mgr->dqs_ena);
  304. }
  305. /* load up dqs io config settings */
  306. static void scc_mgr_load_dqs_io(void)
  307. {
  308. writel(0, &sdr_scc_mgr->dqs_io_ena);
  309. }
  310. /* load up dq config settings */
  311. static void scc_mgr_load_dq(uint32_t dq_in_group)
  312. {
  313. writel(dq_in_group, &sdr_scc_mgr->dq_ena);
  314. }
  315. /* load up dm config settings */
  316. static void scc_mgr_load_dm(uint32_t dm)
  317. {
  318. writel(dm, &sdr_scc_mgr->dm_ena);
  319. }
  320. /**
  321. * scc_mgr_set_all_ranks() - Set SCC Manager register for all ranks
  322. * @off: Base offset in SCC Manager space
  323. * @grp: Read/Write group
  324. * @val: Value to be set
  325. * @update: If non-zero, trigger SCC Manager update for all ranks
  326. *
  327. * This function sets the SCC Manager (Scan Chain Control Manager) register
  328. * and optionally triggers the SCC update for all ranks.
  329. */
  330. static void scc_mgr_set_all_ranks(const u32 off, const u32 grp, const u32 val,
  331. const int update)
  332. {
  333. u32 r;
  334. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
  335. r += NUM_RANKS_PER_SHADOW_REG) {
  336. scc_mgr_set(off, grp, val);
  337. if (update || (r == 0)) {
  338. writel(grp, &sdr_scc_mgr->dqs_ena);
  339. writel(0, &sdr_scc_mgr->update);
  340. }
  341. }
  342. }
  343. static void scc_mgr_set_dqs_en_phase_all_ranks(u32 read_group, u32 phase)
  344. {
  345. /*
  346. * USER although the h/w doesn't support different phases per
  347. * shadow register, for simplicity our scc manager modeling
  348. * keeps different phase settings per shadow reg, and it's
  349. * important for us to keep them in sync to match h/w.
  350. * for efficiency, the scan chain update should occur only
  351. * once to sr0.
  352. */
  353. scc_mgr_set_all_ranks(SCC_MGR_DQS_EN_PHASE_OFFSET,
  354. read_group, phase, 0);
  355. }
  356. static void scc_mgr_set_dqdqs_output_phase_all_ranks(uint32_t write_group,
  357. uint32_t phase)
  358. {
  359. /*
  360. * USER although the h/w doesn't support different phases per
  361. * shadow register, for simplicity our scc manager modeling
  362. * keeps different phase settings per shadow reg, and it's
  363. * important for us to keep them in sync to match h/w.
  364. * for efficiency, the scan chain update should occur only
  365. * once to sr0.
  366. */
  367. scc_mgr_set_all_ranks(SCC_MGR_DQDQS_OUT_PHASE_OFFSET,
  368. write_group, phase, 0);
  369. }
  370. static void scc_mgr_set_dqs_en_delay_all_ranks(uint32_t read_group,
  371. uint32_t delay)
  372. {
  373. /*
  374. * In shadow register mode, the T11 settings are stored in
  375. * registers in the core, which are updated by the DQS_ENA
  376. * signals. Not issuing the SCC_MGR_UPD command allows us to
  377. * save lots of rank switching overhead, by calling
  378. * select_shadow_regs_for_update with update_scan_chains
  379. * set to 0.
  380. */
  381. scc_mgr_set_all_ranks(SCC_MGR_DQS_EN_DELAY_OFFSET,
  382. read_group, delay, 1);
  383. writel(0, &sdr_scc_mgr->update);
  384. }
  385. /**
  386. * scc_mgr_set_oct_out1_delay() - Set OCT output delay
  387. * @write_group: Write group
  388. * @delay: Delay value
  389. *
  390. * This function sets the OCT output delay in SCC manager.
  391. */
  392. static void scc_mgr_set_oct_out1_delay(const u32 write_group, const u32 delay)
  393. {
  394. const int ratio = RW_MGR_MEM_IF_READ_DQS_WIDTH /
  395. RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
  396. const int base = write_group * ratio;
  397. int i;
  398. /*
  399. * Load the setting in the SCC manager
  400. * Although OCT affects only write data, the OCT delay is controlled
  401. * by the DQS logic block which is instantiated once per read group.
  402. * For protocols where a write group consists of multiple read groups,
  403. * the setting must be set multiple times.
  404. */
  405. for (i = 0; i < ratio; i++)
  406. scc_mgr_set(SCC_MGR_OCT_OUT1_DELAY_OFFSET, base + i, delay);
  407. }
  408. /**
  409. * scc_mgr_set_hhp_extras() - Set HHP extras.
  410. *
  411. * Load the fixed setting in the SCC manager HHP extras.
  412. */
  413. static void scc_mgr_set_hhp_extras(void)
  414. {
  415. /*
  416. * Load the fixed setting in the SCC manager
  417. * bits: 0:0 = 1'b1 - DQS bypass
  418. * bits: 1:1 = 1'b1 - DQ bypass
  419. * bits: 4:2 = 3'b001 - rfifo_mode
  420. * bits: 6:5 = 2'b01 - rfifo clock_select
  421. * bits: 7:7 = 1'b0 - separate gating from ungating setting
  422. * bits: 8:8 = 1'b0 - separate OE from Output delay setting
  423. */
  424. const u32 value = (0 << 8) | (0 << 7) | (1 << 5) |
  425. (1 << 2) | (1 << 1) | (1 << 0);
  426. const u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS |
  427. SCC_MGR_HHP_GLOBALS_OFFSET |
  428. SCC_MGR_HHP_EXTRAS_OFFSET;
  429. debug_cond(DLEVEL == 1, "%s:%d Setting HHP Extras\n",
  430. __func__, __LINE__);
  431. writel(value, addr);
  432. debug_cond(DLEVEL == 1, "%s:%d Done Setting HHP Extras\n",
  433. __func__, __LINE__);
  434. }
  435. /**
  436. * scc_mgr_zero_all() - Zero all DQS config
  437. *
  438. * Zero all DQS config.
  439. */
  440. static void scc_mgr_zero_all(void)
  441. {
  442. int i, r;
  443. /*
  444. * USER Zero all DQS config settings, across all groups and all
  445. * shadow registers
  446. */
  447. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
  448. r += NUM_RANKS_PER_SHADOW_REG) {
  449. for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
  450. /*
  451. * The phases actually don't exist on a per-rank basis,
  452. * but there's no harm updating them several times, so
  453. * let's keep the code simple.
  454. */
  455. scc_mgr_set_dqs_bus_in_delay(i, IO_DQS_IN_RESERVE);
  456. scc_mgr_set_dqs_en_phase(i, 0);
  457. scc_mgr_set_dqs_en_delay(i, 0);
  458. }
  459. for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
  460. scc_mgr_set_dqdqs_output_phase(i, 0);
  461. /* Arria V/Cyclone V don't have out2. */
  462. scc_mgr_set_oct_out1_delay(i, IO_DQS_OUT_RESERVE);
  463. }
  464. }
  465. /* Multicast to all DQS group enables. */
  466. writel(0xff, &sdr_scc_mgr->dqs_ena);
  467. writel(0, &sdr_scc_mgr->update);
  468. }
  469. /**
  470. * scc_set_bypass_mode() - Set bypass mode and trigger SCC update
  471. * @write_group: Write group
  472. *
  473. * Set bypass mode and trigger SCC update.
  474. */
  475. static void scc_set_bypass_mode(const u32 write_group)
  476. {
  477. /* Multicast to all DQ enables. */
  478. writel(0xff, &sdr_scc_mgr->dq_ena);
  479. writel(0xff, &sdr_scc_mgr->dm_ena);
  480. /* Update current DQS IO enable. */
  481. writel(0, &sdr_scc_mgr->dqs_io_ena);
  482. /* Update the DQS logic. */
  483. writel(write_group, &sdr_scc_mgr->dqs_ena);
  484. /* Hit update. */
  485. writel(0, &sdr_scc_mgr->update);
  486. }
  487. /**
  488. * scc_mgr_load_dqs_for_write_group() - Load DQS settings for Write Group
  489. * @write_group: Write group
  490. *
  491. * Load DQS settings for Write Group, do not trigger SCC update.
  492. */
  493. static void scc_mgr_load_dqs_for_write_group(const u32 write_group)
  494. {
  495. const int ratio = RW_MGR_MEM_IF_READ_DQS_WIDTH /
  496. RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
  497. const int base = write_group * ratio;
  498. int i;
  499. /*
  500. * Load the setting in the SCC manager
  501. * Although OCT affects only write data, the OCT delay is controlled
  502. * by the DQS logic block which is instantiated once per read group.
  503. * For protocols where a write group consists of multiple read groups,
  504. * the setting must be set multiple times.
  505. */
  506. for (i = 0; i < ratio; i++)
  507. writel(base + i, &sdr_scc_mgr->dqs_ena);
  508. }
  509. /**
  510. * scc_mgr_zero_group() - Zero all configs for a group
  511. *
  512. * Zero DQ, DM, DQS and OCT configs for a group.
  513. */
  514. static void scc_mgr_zero_group(const u32 write_group, const int out_only)
  515. {
  516. int i, r;
  517. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
  518. r += NUM_RANKS_PER_SHADOW_REG) {
  519. /* Zero all DQ config settings. */
  520. for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
  521. scc_mgr_set_dq_out1_delay(i, 0);
  522. if (!out_only)
  523. scc_mgr_set_dq_in_delay(i, 0);
  524. }
  525. /* Multicast to all DQ enables. */
  526. writel(0xff, &sdr_scc_mgr->dq_ena);
  527. /* Zero all DM config settings. */
  528. for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++)
  529. scc_mgr_set_dm_out1_delay(i, 0);
  530. /* Multicast to all DM enables. */
  531. writel(0xff, &sdr_scc_mgr->dm_ena);
  532. /* Zero all DQS IO settings. */
  533. if (!out_only)
  534. scc_mgr_set_dqs_io_in_delay(0);
  535. /* Arria V/Cyclone V don't have out2. */
  536. scc_mgr_set_dqs_out1_delay(IO_DQS_OUT_RESERVE);
  537. scc_mgr_set_oct_out1_delay(write_group, IO_DQS_OUT_RESERVE);
  538. scc_mgr_load_dqs_for_write_group(write_group);
  539. /* Multicast to all DQS IO enables (only 1 in total). */
  540. writel(0, &sdr_scc_mgr->dqs_io_ena);
  541. /* Hit update to zero everything. */
  542. writel(0, &sdr_scc_mgr->update);
  543. }
  544. }
  545. /*
  546. * apply and load a particular input delay for the DQ pins in a group
  547. * group_bgn is the index of the first dq pin (in the write group)
  548. */
  549. static void scc_mgr_apply_group_dq_in_delay(uint32_t group_bgn, uint32_t delay)
  550. {
  551. uint32_t i, p;
  552. for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
  553. scc_mgr_set_dq_in_delay(p, delay);
  554. scc_mgr_load_dq(p);
  555. }
  556. }
  557. /**
  558. * scc_mgr_apply_group_dq_out1_delay() - Apply and load an output delay for the DQ pins in a group
  559. * @delay: Delay value
  560. *
  561. * Apply and load a particular output delay for the DQ pins in a group.
  562. */
  563. static void scc_mgr_apply_group_dq_out1_delay(const u32 delay)
  564. {
  565. int i;
  566. for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
  567. scc_mgr_set_dq_out1_delay(i, delay);
  568. scc_mgr_load_dq(i);
  569. }
  570. }
  571. /* apply and load a particular output delay for the DM pins in a group */
  572. static void scc_mgr_apply_group_dm_out1_delay(uint32_t delay1)
  573. {
  574. uint32_t i;
  575. for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
  576. scc_mgr_set_dm_out1_delay(i, delay1);
  577. scc_mgr_load_dm(i);
  578. }
  579. }
  580. /* apply and load delay on both DQS and OCT out1 */
  581. static void scc_mgr_apply_group_dqs_io_and_oct_out1(uint32_t write_group,
  582. uint32_t delay)
  583. {
  584. scc_mgr_set_dqs_out1_delay(delay);
  585. scc_mgr_load_dqs_io();
  586. scc_mgr_set_oct_out1_delay(write_group, delay);
  587. scc_mgr_load_dqs_for_write_group(write_group);
  588. }
  589. /**
  590. * scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output side: DQ, DM, DQS, OCT
  591. * @write_group: Write group
  592. * @delay: Delay value
  593. *
  594. * Apply a delay to the entire output side: DQ, DM, DQS, OCT.
  595. */
  596. static void scc_mgr_apply_group_all_out_delay_add(const u32 write_group,
  597. const u32 delay)
  598. {
  599. u32 i, new_delay;
  600. /* DQ shift */
  601. for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++)
  602. scc_mgr_load_dq(i);
  603. /* DM shift */
  604. for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++)
  605. scc_mgr_load_dm(i);
  606. /* DQS shift */
  607. new_delay = READ_SCC_DQS_IO_OUT2_DELAY + delay;
  608. if (new_delay > IO_IO_OUT2_DELAY_MAX) {
  609. debug_cond(DLEVEL == 1,
  610. "%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n",
  611. __func__, __LINE__, write_group, delay, new_delay,
  612. IO_IO_OUT2_DELAY_MAX,
  613. new_delay - IO_IO_OUT2_DELAY_MAX);
  614. new_delay -= IO_IO_OUT2_DELAY_MAX;
  615. scc_mgr_set_dqs_out1_delay(new_delay);
  616. }
  617. scc_mgr_load_dqs_io();
  618. /* OCT shift */
  619. new_delay = READ_SCC_OCT_OUT2_DELAY + delay;
  620. if (new_delay > IO_IO_OUT2_DELAY_MAX) {
  621. debug_cond(DLEVEL == 1,
  622. "%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n",
  623. __func__, __LINE__, write_group, delay,
  624. new_delay, IO_IO_OUT2_DELAY_MAX,
  625. new_delay - IO_IO_OUT2_DELAY_MAX);
  626. new_delay -= IO_IO_OUT2_DELAY_MAX;
  627. scc_mgr_set_oct_out1_delay(write_group, new_delay);
  628. }
  629. scc_mgr_load_dqs_for_write_group(write_group);
  630. }
  631. /**
  632. * scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output side to all ranks
  633. * @write_group: Write group
  634. * @delay: Delay value
  635. *
  636. * Apply a delay to the entire output side (DQ, DM, DQS, OCT) to all ranks.
  637. */
  638. static void
  639. scc_mgr_apply_group_all_out_delay_add_all_ranks(const u32 write_group,
  640. const u32 delay)
  641. {
  642. int r;
  643. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
  644. r += NUM_RANKS_PER_SHADOW_REG) {
  645. scc_mgr_apply_group_all_out_delay_add(write_group, delay);
  646. writel(0, &sdr_scc_mgr->update);
  647. }
  648. }
  649. /**
  650. * set_jump_as_return() - Return instruction optimization
  651. *
  652. * Optimization used to recover some slots in ddr3 inst_rom could be
  653. * applied to other protocols if we wanted to
  654. */
  655. static void set_jump_as_return(void)
  656. {
  657. /*
  658. * To save space, we replace return with jump to special shared
  659. * RETURN instruction so we set the counter to large value so that
  660. * we always jump.
  661. */
  662. writel(0xff, &sdr_rw_load_mgr_regs->load_cntr0);
  663. writel(RW_MGR_RETURN, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  664. }
  665. /*
  666. * should always use constants as argument to ensure all computations are
  667. * performed at compile time
  668. */
  669. static void delay_for_n_mem_clocks(const uint32_t clocks)
  670. {
  671. uint32_t afi_clocks;
  672. uint8_t inner = 0;
  673. uint8_t outer = 0;
  674. uint16_t c_loop = 0;
  675. debug("%s:%d: clocks=%u ... start\n", __func__, __LINE__, clocks);
  676. afi_clocks = (clocks + AFI_RATE_RATIO-1) / AFI_RATE_RATIO;
  677. /* scale (rounding up) to get afi clocks */
  678. /*
  679. * Note, we don't bother accounting for being off a little bit
  680. * because of a few extra instructions in outer loops
  681. * Note, the loops have a test at the end, and do the test before
  682. * the decrement, and so always perform the loop
  683. * 1 time more than the counter value
  684. */
  685. if (afi_clocks == 0) {
  686. ;
  687. } else if (afi_clocks <= 0x100) {
  688. inner = afi_clocks-1;
  689. outer = 0;
  690. c_loop = 0;
  691. } else if (afi_clocks <= 0x10000) {
  692. inner = 0xff;
  693. outer = (afi_clocks-1) >> 8;
  694. c_loop = 0;
  695. } else {
  696. inner = 0xff;
  697. outer = 0xff;
  698. c_loop = (afi_clocks-1) >> 16;
  699. }
  700. /*
  701. * rom instructions are structured as follows:
  702. *
  703. * IDLE_LOOP2: jnz cntr0, TARGET_A
  704. * IDLE_LOOP1: jnz cntr1, TARGET_B
  705. * return
  706. *
  707. * so, when doing nested loops, TARGET_A is set to IDLE_LOOP2, and
  708. * TARGET_B is set to IDLE_LOOP2 as well
  709. *
  710. * if we have no outer loop, though, then we can use IDLE_LOOP1 only,
  711. * and set TARGET_B to IDLE_LOOP1 and we skip IDLE_LOOP2 entirely
  712. *
  713. * a little confusing, but it helps save precious space in the inst_rom
  714. * and sequencer rom and keeps the delays more accurate and reduces
  715. * overhead
  716. */
  717. if (afi_clocks <= 0x100) {
  718. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
  719. &sdr_rw_load_mgr_regs->load_cntr1);
  720. writel(RW_MGR_IDLE_LOOP1,
  721. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  722. writel(RW_MGR_IDLE_LOOP1, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  723. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  724. } else {
  725. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
  726. &sdr_rw_load_mgr_regs->load_cntr0);
  727. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(outer),
  728. &sdr_rw_load_mgr_regs->load_cntr1);
  729. writel(RW_MGR_IDLE_LOOP2,
  730. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  731. writel(RW_MGR_IDLE_LOOP2,
  732. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  733. /* hack to get around compiler not being smart enough */
  734. if (afi_clocks <= 0x10000) {
  735. /* only need to run once */
  736. writel(RW_MGR_IDLE_LOOP2, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  737. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  738. } else {
  739. do {
  740. writel(RW_MGR_IDLE_LOOP2,
  741. SDR_PHYGRP_RWMGRGRP_ADDRESS |
  742. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  743. } while (c_loop-- != 0);
  744. }
  745. }
  746. debug("%s:%d clocks=%u ... end\n", __func__, __LINE__, clocks);
  747. }
  748. /**
  749. * rw_mgr_mem_init_load_regs() - Load instruction registers
  750. * @cntr0: Counter 0 value
  751. * @cntr1: Counter 1 value
  752. * @cntr2: Counter 2 value
  753. * @jump: Jump instruction value
  754. *
  755. * Load instruction registers.
  756. */
  757. static void rw_mgr_mem_init_load_regs(u32 cntr0, u32 cntr1, u32 cntr2, u32 jump)
  758. {
  759. uint32_t grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  760. RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  761. /* Load counters */
  762. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr0),
  763. &sdr_rw_load_mgr_regs->load_cntr0);
  764. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr1),
  765. &sdr_rw_load_mgr_regs->load_cntr1);
  766. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr2),
  767. &sdr_rw_load_mgr_regs->load_cntr2);
  768. /* Load jump address */
  769. writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  770. writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  771. writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  772. /* Execute count instruction */
  773. writel(jump, grpaddr);
  774. }
  775. /**
  776. * rw_mgr_mem_load_user() - Load user calibration values
  777. * @fin1: Final instruction 1
  778. * @fin2: Final instruction 2
  779. * @precharge: If 1, precharge the banks at the end
  780. *
  781. * Load user calibration values and optionally precharge the banks.
  782. */
  783. static void rw_mgr_mem_load_user(const u32 fin1, const u32 fin2,
  784. const int precharge)
  785. {
  786. u32 grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  787. RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  788. u32 r;
  789. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
  790. if (param->skip_ranks[r]) {
  791. /* request to skip the rank */
  792. continue;
  793. }
  794. /* set rank */
  795. set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
  796. /* precharge all banks ... */
  797. if (precharge)
  798. writel(RW_MGR_PRECHARGE_ALL, grpaddr);
  799. /*
  800. * USER Use Mirror-ed commands for odd ranks if address
  801. * mirrorring is on
  802. */
  803. if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) {
  804. set_jump_as_return();
  805. writel(RW_MGR_MRS2_MIRR, grpaddr);
  806. delay_for_n_mem_clocks(4);
  807. set_jump_as_return();
  808. writel(RW_MGR_MRS3_MIRR, grpaddr);
  809. delay_for_n_mem_clocks(4);
  810. set_jump_as_return();
  811. writel(RW_MGR_MRS1_MIRR, grpaddr);
  812. delay_for_n_mem_clocks(4);
  813. set_jump_as_return();
  814. writel(fin1, grpaddr);
  815. } else {
  816. set_jump_as_return();
  817. writel(RW_MGR_MRS2, grpaddr);
  818. delay_for_n_mem_clocks(4);
  819. set_jump_as_return();
  820. writel(RW_MGR_MRS3, grpaddr);
  821. delay_for_n_mem_clocks(4);
  822. set_jump_as_return();
  823. writel(RW_MGR_MRS1, grpaddr);
  824. set_jump_as_return();
  825. writel(fin2, grpaddr);
  826. }
  827. if (precharge)
  828. continue;
  829. set_jump_as_return();
  830. writel(RW_MGR_ZQCL, grpaddr);
  831. /* tZQinit = tDLLK = 512 ck cycles */
  832. delay_for_n_mem_clocks(512);
  833. }
  834. }
  835. /**
  836. * rw_mgr_mem_initialize() - Initialize RW Manager
  837. *
  838. * Initialize RW Manager.
  839. */
  840. static void rw_mgr_mem_initialize(void)
  841. {
  842. debug("%s:%d\n", __func__, __LINE__);
  843. /* The reset / cke part of initialization is broadcasted to all ranks */
  844. writel(RW_MGR_RANK_ALL, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  845. RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
  846. /*
  847. * Here's how you load register for a loop
  848. * Counters are located @ 0x800
  849. * Jump address are located @ 0xC00
  850. * For both, registers 0 to 3 are selected using bits 3 and 2, like
  851. * in 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C
  852. * I know this ain't pretty, but Avalon bus throws away the 2 least
  853. * significant bits
  854. */
  855. /* Start with memory RESET activated */
  856. /* tINIT = 200us */
  857. /*
  858. * 200us @ 266MHz (3.75 ns) ~ 54000 clock cycles
  859. * If a and b are the number of iteration in 2 nested loops
  860. * it takes the following number of cycles to complete the operation:
  861. * number_of_cycles = ((2 + n) * a + 2) * b
  862. * where n is the number of instruction in the inner loop
  863. * One possible solution is n = 0 , a = 256 , b = 106 => a = FF,
  864. * b = 6A
  865. */
  866. rw_mgr_mem_init_load_regs(SEQ_TINIT_CNTR0_VAL, SEQ_TINIT_CNTR1_VAL,
  867. SEQ_TINIT_CNTR2_VAL,
  868. RW_MGR_INIT_RESET_0_CKE_0);
  869. /* Indicate that memory is stable. */
  870. writel(1, &phy_mgr_cfg->reset_mem_stbl);
  871. /*
  872. * transition the RESET to high
  873. * Wait for 500us
  874. */
  875. /*
  876. * 500us @ 266MHz (3.75 ns) ~ 134000 clock cycles
  877. * If a and b are the number of iteration in 2 nested loops
  878. * it takes the following number of cycles to complete the operation
  879. * number_of_cycles = ((2 + n) * a + 2) * b
  880. * where n is the number of instruction in the inner loop
  881. * One possible solution is n = 2 , a = 131 , b = 256 => a = 83,
  882. * b = FF
  883. */
  884. rw_mgr_mem_init_load_regs(SEQ_TRESET_CNTR0_VAL, SEQ_TRESET_CNTR1_VAL,
  885. SEQ_TRESET_CNTR2_VAL,
  886. RW_MGR_INIT_RESET_1_CKE_0);
  887. /* Bring up clock enable. */
  888. /* tXRP < 250 ck cycles */
  889. delay_for_n_mem_clocks(250);
  890. rw_mgr_mem_load_user(RW_MGR_MRS0_DLL_RESET_MIRR, RW_MGR_MRS0_DLL_RESET,
  891. 0);
  892. }
  893. /*
  894. * At the end of calibration we have to program the user settings in, and
  895. * USER hand off the memory to the user.
  896. */
  897. static void rw_mgr_mem_handoff(void)
  898. {
  899. rw_mgr_mem_load_user(RW_MGR_MRS0_USER_MIRR, RW_MGR_MRS0_USER, 1);
  900. /*
  901. * USER need to wait tMOD (12CK or 15ns) time before issuing
  902. * other commands, but we will have plenty of NIOS cycles before
  903. * actual handoff so its okay.
  904. */
  905. }
  906. /**
  907. * rw_mgr_mem_calibrate_read_test_patterns() - Read back test patterns
  908. * @rank_bgn: Rank number
  909. * @group: Read/Write Group
  910. * @all_ranks: Test all ranks
  911. *
  912. * Performs a guaranteed read on the patterns we are going to use during a
  913. * read test to ensure memory works.
  914. */
  915. static int
  916. rw_mgr_mem_calibrate_read_test_patterns(const u32 rank_bgn, const u32 group,
  917. const u32 all_ranks)
  918. {
  919. const u32 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  920. RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  921. const u32 addr_offset =
  922. (group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS) << 2;
  923. const u32 rank_end = all_ranks ?
  924. RW_MGR_MEM_NUMBER_OF_RANKS :
  925. (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
  926. const u32 shift_ratio = RW_MGR_MEM_DQ_PER_READ_DQS /
  927. RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS;
  928. const u32 correct_mask_vg = param->read_correct_mask_vg;
  929. u32 tmp_bit_chk, base_rw_mgr, bit_chk;
  930. int vg, r;
  931. int ret = 0;
  932. bit_chk = param->read_correct_mask;
  933. for (r = rank_bgn; r < rank_end; r++) {
  934. /* Request to skip the rank */
  935. if (param->skip_ranks[r])
  936. continue;
  937. /* Set rank */
  938. set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
  939. /* Load up a constant bursts of read commands */
  940. writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
  941. writel(RW_MGR_GUARANTEED_READ,
  942. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  943. writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
  944. writel(RW_MGR_GUARANTEED_READ_CONT,
  945. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  946. tmp_bit_chk = 0;
  947. for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS - 1;
  948. vg >= 0; vg--) {
  949. /* Reset the FIFOs to get pointers to known state. */
  950. writel(0, &phy_mgr_cmd->fifo_reset);
  951. writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  952. RW_MGR_RESET_READ_DATAPATH_OFFSET);
  953. writel(RW_MGR_GUARANTEED_READ,
  954. addr + addr_offset + (vg << 2));
  955. base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
  956. tmp_bit_chk <<= shift_ratio;
  957. tmp_bit_chk |= correct_mask_vg & ~base_rw_mgr;
  958. }
  959. bit_chk &= tmp_bit_chk;
  960. }
  961. writel(RW_MGR_CLEAR_DQS_ENABLE, addr + (group << 2));
  962. set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
  963. if (bit_chk != param->read_correct_mask)
  964. ret = -EIO;
  965. debug_cond(DLEVEL == 1,
  966. "%s:%d test_load_patterns(%u,ALL) => (%u == %u) => %i\n",
  967. __func__, __LINE__, group, bit_chk,
  968. param->read_correct_mask, ret);
  969. return ret;
  970. }
  971. /**
  972. * rw_mgr_mem_calibrate_read_load_patterns() - Load up the patterns for read test
  973. * @rank_bgn: Rank number
  974. * @all_ranks: Test all ranks
  975. *
  976. * Load up the patterns we are going to use during a read test.
  977. */
  978. static void rw_mgr_mem_calibrate_read_load_patterns(const u32 rank_bgn,
  979. const int all_ranks)
  980. {
  981. const u32 rank_end = all_ranks ?
  982. RW_MGR_MEM_NUMBER_OF_RANKS :
  983. (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
  984. u32 r;
  985. debug("%s:%d\n", __func__, __LINE__);
  986. for (r = rank_bgn; r < rank_end; r++) {
  987. if (param->skip_ranks[r])
  988. /* request to skip the rank */
  989. continue;
  990. /* set rank */
  991. set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
  992. /* Load up a constant bursts */
  993. writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
  994. writel(RW_MGR_GUARANTEED_WRITE_WAIT0,
  995. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  996. writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
  997. writel(RW_MGR_GUARANTEED_WRITE_WAIT1,
  998. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  999. writel(0x04, &sdr_rw_load_mgr_regs->load_cntr2);
  1000. writel(RW_MGR_GUARANTEED_WRITE_WAIT2,
  1001. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1002. writel(0x04, &sdr_rw_load_mgr_regs->load_cntr3);
  1003. writel(RW_MGR_GUARANTEED_WRITE_WAIT3,
  1004. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  1005. writel(RW_MGR_GUARANTEED_WRITE, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1006. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  1007. }
  1008. set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
  1009. }
  1010. /*
  1011. * try a read and see if it returns correct data back. has dummy reads
  1012. * inserted into the mix used to align dqs enable. has more thorough checks
  1013. * than the regular read test.
  1014. */
  1015. static uint32_t rw_mgr_mem_calibrate_read_test(uint32_t rank_bgn, uint32_t group,
  1016. uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
  1017. uint32_t all_groups, uint32_t all_ranks)
  1018. {
  1019. uint32_t r, vg;
  1020. uint32_t correct_mask_vg;
  1021. uint32_t tmp_bit_chk;
  1022. uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
  1023. (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
  1024. uint32_t addr;
  1025. uint32_t base_rw_mgr;
  1026. *bit_chk = param->read_correct_mask;
  1027. correct_mask_vg = param->read_correct_mask_vg;
  1028. uint32_t quick_read_mode = (((STATIC_CALIB_STEPS) &
  1029. CALIB_SKIP_DELAY_SWEEPS) && ENABLE_SUPER_QUICK_CALIBRATION);
  1030. for (r = rank_bgn; r < rank_end; r++) {
  1031. if (param->skip_ranks[r])
  1032. /* request to skip the rank */
  1033. continue;
  1034. /* set rank */
  1035. set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
  1036. writel(0x10, &sdr_rw_load_mgr_regs->load_cntr1);
  1037. writel(RW_MGR_READ_B2B_WAIT1,
  1038. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  1039. writel(0x10, &sdr_rw_load_mgr_regs->load_cntr2);
  1040. writel(RW_MGR_READ_B2B_WAIT2,
  1041. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1042. if (quick_read_mode)
  1043. writel(0x1, &sdr_rw_load_mgr_regs->load_cntr0);
  1044. /* need at least two (1+1) reads to capture failures */
  1045. else if (all_groups)
  1046. writel(0x06, &sdr_rw_load_mgr_regs->load_cntr0);
  1047. else
  1048. writel(0x32, &sdr_rw_load_mgr_regs->load_cntr0);
  1049. writel(RW_MGR_READ_B2B,
  1050. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  1051. if (all_groups)
  1052. writel(RW_MGR_MEM_IF_READ_DQS_WIDTH *
  1053. RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS - 1,
  1054. &sdr_rw_load_mgr_regs->load_cntr3);
  1055. else
  1056. writel(0x0, &sdr_rw_load_mgr_regs->load_cntr3);
  1057. writel(RW_MGR_READ_B2B,
  1058. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  1059. tmp_bit_chk = 0;
  1060. for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) {
  1061. /* reset the fifos to get pointers to known state */
  1062. writel(0, &phy_mgr_cmd->fifo_reset);
  1063. writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1064. RW_MGR_RESET_READ_DATAPATH_OFFSET);
  1065. tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS
  1066. / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS);
  1067. if (all_groups)
  1068. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_ALL_GROUPS_OFFSET;
  1069. else
  1070. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  1071. writel(RW_MGR_READ_B2B, addr +
  1072. ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS +
  1073. vg) << 2));
  1074. base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
  1075. tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
  1076. if (vg == 0)
  1077. break;
  1078. }
  1079. *bit_chk &= tmp_bit_chk;
  1080. }
  1081. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  1082. writel(RW_MGR_CLEAR_DQS_ENABLE, addr + (group << 2));
  1083. if (all_correct) {
  1084. set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
  1085. debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ALL,%u) =>\
  1086. (%u == %u) => %lu", __func__, __LINE__, group,
  1087. all_groups, *bit_chk, param->read_correct_mask,
  1088. (long unsigned int)(*bit_chk ==
  1089. param->read_correct_mask));
  1090. return *bit_chk == param->read_correct_mask;
  1091. } else {
  1092. set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
  1093. debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ONE,%u) =>\
  1094. (%u != %lu) => %lu\n", __func__, __LINE__,
  1095. group, all_groups, *bit_chk, (long unsigned int)0,
  1096. (long unsigned int)(*bit_chk != 0x00));
  1097. return *bit_chk != 0x00;
  1098. }
  1099. }
  1100. static uint32_t rw_mgr_mem_calibrate_read_test_all_ranks(uint32_t group,
  1101. uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
  1102. uint32_t all_groups)
  1103. {
  1104. return rw_mgr_mem_calibrate_read_test(0, group, num_tries, all_correct,
  1105. bit_chk, all_groups, 1);
  1106. }
  1107. static void rw_mgr_incr_vfifo(uint32_t grp, uint32_t *v)
  1108. {
  1109. writel(grp, &phy_mgr_cmd->inc_vfifo_hard_phy);
  1110. (*v)++;
  1111. }
  1112. static void rw_mgr_decr_vfifo(uint32_t grp, uint32_t *v)
  1113. {
  1114. uint32_t i;
  1115. for (i = 0; i < VFIFO_SIZE-1; i++)
  1116. rw_mgr_incr_vfifo(grp, v);
  1117. }
  1118. static int find_vfifo_read(uint32_t grp, uint32_t *bit_chk)
  1119. {
  1120. uint32_t v;
  1121. uint32_t fail_cnt = 0;
  1122. uint32_t test_status;
  1123. for (v = 0; v < VFIFO_SIZE; ) {
  1124. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo %u\n",
  1125. __func__, __LINE__, v);
  1126. test_status = rw_mgr_mem_calibrate_read_test_all_ranks
  1127. (grp, 1, PASS_ONE_BIT, bit_chk, 0);
  1128. if (!test_status) {
  1129. fail_cnt++;
  1130. if (fail_cnt == 2)
  1131. break;
  1132. }
  1133. /* fiddle with FIFO */
  1134. rw_mgr_incr_vfifo(grp, &v);
  1135. }
  1136. if (v >= VFIFO_SIZE) {
  1137. /* no failing read found!! Something must have gone wrong */
  1138. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo failed\n",
  1139. __func__, __LINE__);
  1140. return 0;
  1141. } else {
  1142. return v;
  1143. }
  1144. }
  1145. static int sdr_working_phase(uint32_t grp, uint32_t *bit_chk,
  1146. uint32_t dtaps_per_ptap, uint32_t *work_bgn,
  1147. uint32_t *v, uint32_t *d, uint32_t *p,
  1148. uint32_t *i, uint32_t *max_working_cnt)
  1149. {
  1150. uint32_t found_begin = 0;
  1151. uint32_t tmp_delay = 0;
  1152. uint32_t test_status;
  1153. for (*d = 0; *d <= dtaps_per_ptap; (*d)++, tmp_delay +=
  1154. IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
  1155. *work_bgn = tmp_delay;
  1156. scc_mgr_set_dqs_en_delay_all_ranks(grp, *d);
  1157. for (*i = 0; *i < VFIFO_SIZE; (*i)++) {
  1158. for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_bgn +=
  1159. IO_DELAY_PER_OPA_TAP) {
  1160. scc_mgr_set_dqs_en_phase_all_ranks(grp, *p);
  1161. test_status =
  1162. rw_mgr_mem_calibrate_read_test_all_ranks
  1163. (grp, 1, PASS_ONE_BIT, bit_chk, 0);
  1164. if (test_status) {
  1165. *max_working_cnt = 1;
  1166. found_begin = 1;
  1167. break;
  1168. }
  1169. }
  1170. if (found_begin)
  1171. break;
  1172. if (*p > IO_DQS_EN_PHASE_MAX)
  1173. /* fiddle with FIFO */
  1174. rw_mgr_incr_vfifo(grp, v);
  1175. }
  1176. if (found_begin)
  1177. break;
  1178. }
  1179. if (*i >= VFIFO_SIZE) {
  1180. /* cannot find working solution */
  1181. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: no vfifo/\
  1182. ptap/dtap\n", __func__, __LINE__);
  1183. return 0;
  1184. } else {
  1185. return 1;
  1186. }
  1187. }
  1188. static void sdr_backup_phase(uint32_t grp, uint32_t *bit_chk,
  1189. uint32_t *work_bgn, uint32_t *v, uint32_t *d,
  1190. uint32_t *p, uint32_t *max_working_cnt)
  1191. {
  1192. uint32_t found_begin = 0;
  1193. uint32_t tmp_delay;
  1194. /* Special case code for backing up a phase */
  1195. if (*p == 0) {
  1196. *p = IO_DQS_EN_PHASE_MAX;
  1197. rw_mgr_decr_vfifo(grp, v);
  1198. } else {
  1199. (*p)--;
  1200. }
  1201. tmp_delay = *work_bgn - IO_DELAY_PER_OPA_TAP;
  1202. scc_mgr_set_dqs_en_phase_all_ranks(grp, *p);
  1203. for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_bgn;
  1204. (*d)++, tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
  1205. scc_mgr_set_dqs_en_delay_all_ranks(grp, *d);
  1206. if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
  1207. PASS_ONE_BIT,
  1208. bit_chk, 0)) {
  1209. found_begin = 1;
  1210. *work_bgn = tmp_delay;
  1211. break;
  1212. }
  1213. }
  1214. /* We have found a working dtap before the ptap found above */
  1215. if (found_begin == 1)
  1216. (*max_working_cnt)++;
  1217. /*
  1218. * Restore VFIFO to old state before we decremented it
  1219. * (if needed).
  1220. */
  1221. (*p)++;
  1222. if (*p > IO_DQS_EN_PHASE_MAX) {
  1223. *p = 0;
  1224. rw_mgr_incr_vfifo(grp, v);
  1225. }
  1226. scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);
  1227. }
  1228. static int sdr_nonworking_phase(uint32_t grp, uint32_t *bit_chk,
  1229. uint32_t *work_bgn, uint32_t *v, uint32_t *d,
  1230. uint32_t *p, uint32_t *i, uint32_t *max_working_cnt,
  1231. uint32_t *work_end)
  1232. {
  1233. uint32_t found_end = 0;
  1234. (*p)++;
  1235. *work_end += IO_DELAY_PER_OPA_TAP;
  1236. if (*p > IO_DQS_EN_PHASE_MAX) {
  1237. /* fiddle with FIFO */
  1238. *p = 0;
  1239. rw_mgr_incr_vfifo(grp, v);
  1240. }
  1241. for (; *i < VFIFO_SIZE + 1; (*i)++) {
  1242. for (; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_end
  1243. += IO_DELAY_PER_OPA_TAP) {
  1244. scc_mgr_set_dqs_en_phase_all_ranks(grp, *p);
  1245. if (!rw_mgr_mem_calibrate_read_test_all_ranks
  1246. (grp, 1, PASS_ONE_BIT, bit_chk, 0)) {
  1247. found_end = 1;
  1248. break;
  1249. } else {
  1250. (*max_working_cnt)++;
  1251. }
  1252. }
  1253. if (found_end)
  1254. break;
  1255. if (*p > IO_DQS_EN_PHASE_MAX) {
  1256. /* fiddle with FIFO */
  1257. rw_mgr_incr_vfifo(grp, v);
  1258. *p = 0;
  1259. }
  1260. }
  1261. if (*i >= VFIFO_SIZE + 1) {
  1262. /* cannot see edge of failing read */
  1263. debug_cond(DLEVEL == 2, "%s:%d sdr_nonworking_phase: end:\
  1264. failed\n", __func__, __LINE__);
  1265. return 0;
  1266. } else {
  1267. return 1;
  1268. }
  1269. }
  1270. /**
  1271. * sdr_find_window_center() - Find center of the working DQS window.
  1272. * @grp: Read/Write group
  1273. * @work_bgn: First working settings
  1274. * @work_end: Last working settings
  1275. * @val: VFIFO value
  1276. *
  1277. * Find center of the working DQS enable window.
  1278. */
  1279. static int sdr_find_window_center(const u32 grp, const u32 work_bgn,
  1280. const u32 work_end, const u32 val)
  1281. {
  1282. u32 bit_chk, work_mid, v = val;
  1283. int tmp_delay = 0;
  1284. int i, p, d;
  1285. work_mid = (work_bgn + work_end) / 2;
  1286. debug_cond(DLEVEL == 2, "work_bgn=%d work_end=%d work_mid=%d\n",
  1287. work_bgn, work_end, work_mid);
  1288. /* Get the middle delay to be less than a VFIFO delay */
  1289. tmp_delay = (IO_DQS_EN_PHASE_MAX + 1) * IO_DELAY_PER_OPA_TAP;
  1290. debug_cond(DLEVEL == 2, "vfifo ptap delay %d\n", tmp_delay);
  1291. work_mid %= tmp_delay;
  1292. debug_cond(DLEVEL == 2, "new work_mid %d\n", work_mid);
  1293. tmp_delay = rounddown(work_mid, IO_DELAY_PER_OPA_TAP);
  1294. if (tmp_delay > IO_DQS_EN_PHASE_MAX * IO_DELAY_PER_OPA_TAP)
  1295. tmp_delay = IO_DQS_EN_PHASE_MAX * IO_DELAY_PER_OPA_TAP;
  1296. p = tmp_delay / IO_DELAY_PER_OPA_TAP;
  1297. debug_cond(DLEVEL == 2, "new p %d, tmp_delay=%d\n", p, tmp_delay);
  1298. d = DIV_ROUND_UP(work_mid - tmp_delay, IO_DELAY_PER_DQS_EN_DCHAIN_TAP);
  1299. if (d > IO_DQS_EN_DELAY_MAX)
  1300. d = IO_DQS_EN_DELAY_MAX;
  1301. tmp_delay += d * IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
  1302. debug_cond(DLEVEL == 2, "new d %d, tmp_delay=%d\n", d, tmp_delay);
  1303. scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
  1304. scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
  1305. /*
  1306. * push vfifo until we can successfully calibrate. We can do this
  1307. * because the largest possible margin in 1 VFIFO cycle.
  1308. */
  1309. for (i = 0; i < VFIFO_SIZE; i++) {
  1310. debug_cond(DLEVEL == 2, "find_dqs_en_phase: center: vfifo=%u\n",
  1311. v);
  1312. if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
  1313. PASS_ONE_BIT,
  1314. &bit_chk, 0)) {
  1315. debug_cond(DLEVEL == 2,
  1316. "%s:%d center: found: vfifo=%u ptap=%u dtap=%u\n",
  1317. __func__, __LINE__, v, p, d);
  1318. return 0;
  1319. }
  1320. /* Fiddle with FIFO. */
  1321. rw_mgr_incr_vfifo(grp, &v);
  1322. }
  1323. debug_cond(DLEVEL == 2, "%s:%d center: failed.\n",
  1324. __func__, __LINE__);
  1325. return -EINVAL;
  1326. }
  1327. /* find a good dqs enable to use */
  1328. static uint32_t rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(uint32_t grp)
  1329. {
  1330. uint32_t v, d, p, i;
  1331. uint32_t max_working_cnt;
  1332. uint32_t bit_chk;
  1333. uint32_t dtaps_per_ptap;
  1334. uint32_t work_bgn, work_end;
  1335. uint32_t found_passing_read, found_failing_read, initial_failing_dtap;
  1336. debug("%s:%d %u\n", __func__, __LINE__, grp);
  1337. reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
  1338. scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);
  1339. scc_mgr_set_dqs_en_phase_all_ranks(grp, 0);
  1340. /* ************************************************************** */
  1341. /* * Step 0 : Determine number of delay taps for each phase tap * */
  1342. dtaps_per_ptap = IO_DELAY_PER_OPA_TAP/IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
  1343. /* ********************************************************* */
  1344. /* * Step 1 : First push vfifo until we get a failing read * */
  1345. v = find_vfifo_read(grp, &bit_chk);
  1346. max_working_cnt = 0;
  1347. /* ******************************************************** */
  1348. /* * step 2: find first working phase, increment in ptaps * */
  1349. work_bgn = 0;
  1350. if (sdr_working_phase(grp, &bit_chk, dtaps_per_ptap, &work_bgn, &v, &d,
  1351. &p, &i, &max_working_cnt) == 0)
  1352. return 0;
  1353. work_end = work_bgn;
  1354. /*
  1355. * If d is 0 then the working window covers a phase tap and
  1356. * we can follow the old procedure otherwise, we've found the beginning,
  1357. * and we need to increment the dtaps until we find the end.
  1358. */
  1359. if (d == 0) {
  1360. /* ********************************************************* */
  1361. /* * step 3a: if we have room, back off by one and
  1362. increment in dtaps * */
  1363. sdr_backup_phase(grp, &bit_chk, &work_bgn, &v, &d, &p,
  1364. &max_working_cnt);
  1365. /* ********************************************************* */
  1366. /* * step 4a: go forward from working phase to non working
  1367. phase, increment in ptaps * */
  1368. if (sdr_nonworking_phase(grp, &bit_chk, &work_bgn, &v, &d, &p,
  1369. &i, &max_working_cnt, &work_end) == 0)
  1370. return 0;
  1371. /* ********************************************************* */
  1372. /* * step 5a: back off one from last, increment in dtaps * */
  1373. /* Special case code for backing up a phase */
  1374. if (p == 0) {
  1375. p = IO_DQS_EN_PHASE_MAX;
  1376. rw_mgr_decr_vfifo(grp, &v);
  1377. } else {
  1378. p = p - 1;
  1379. }
  1380. work_end -= IO_DELAY_PER_OPA_TAP;
  1381. scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
  1382. /* * The actual increment of dtaps is done outside of
  1383. the if/else loop to share code */
  1384. d = 0;
  1385. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p: \
  1386. vfifo=%u ptap=%u\n", __func__, __LINE__,
  1387. v, p);
  1388. } else {
  1389. /* ******************************************************* */
  1390. /* * step 3-5b: Find the right edge of the window using
  1391. delay taps * */
  1392. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase:vfifo=%u \
  1393. ptap=%u dtap=%u bgn=%u\n", __func__, __LINE__,
  1394. v, p, d, work_bgn);
  1395. work_end = work_bgn;
  1396. /* * The actual increment of dtaps is done outside of the
  1397. if/else loop to share code */
  1398. /* Only here to counterbalance a subtract later on which is
  1399. not needed if this branch of the algorithm is taken */
  1400. max_working_cnt++;
  1401. }
  1402. /* The dtap increment to find the failing edge is done here */
  1403. for (; d <= IO_DQS_EN_DELAY_MAX; d++, work_end +=
  1404. IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
  1405. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \
  1406. end-2: dtap=%u\n", __func__, __LINE__, d);
  1407. scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
  1408. if (!rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
  1409. PASS_ONE_BIT,
  1410. &bit_chk, 0)) {
  1411. break;
  1412. }
  1413. }
  1414. /* Go back to working dtap */
  1415. if (d != 0)
  1416. work_end -= IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
  1417. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p/d: vfifo=%u \
  1418. ptap=%u dtap=%u end=%u\n", __func__, __LINE__,
  1419. v, p, d-1, work_end);
  1420. if (work_end < work_bgn) {
  1421. /* nil range */
  1422. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: end-2: \
  1423. failed\n", __func__, __LINE__);
  1424. return 0;
  1425. }
  1426. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: found range [%u,%u]\n",
  1427. __func__, __LINE__, work_bgn, work_end);
  1428. /* *************************************************************** */
  1429. /*
  1430. * * We need to calculate the number of dtaps that equal a ptap
  1431. * * To do that we'll back up a ptap and re-find the edge of the
  1432. * * window using dtaps
  1433. */
  1434. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: calculate dtaps_per_ptap \
  1435. for tracking\n", __func__, __LINE__);
  1436. /* Special case code for backing up a phase */
  1437. if (p == 0) {
  1438. p = IO_DQS_EN_PHASE_MAX;
  1439. rw_mgr_decr_vfifo(grp, &v);
  1440. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \
  1441. cycle/phase: v=%u p=%u\n", __func__, __LINE__,
  1442. v, p);
  1443. } else {
  1444. p = p - 1;
  1445. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \
  1446. phase only: v=%u p=%u", __func__, __LINE__,
  1447. v, p);
  1448. }
  1449. scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
  1450. /*
  1451. * Increase dtap until we first see a passing read (in case the
  1452. * window is smaller than a ptap),
  1453. * and then a failing read to mark the edge of the window again
  1454. */
  1455. /* Find a passing read */
  1456. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find passing read\n",
  1457. __func__, __LINE__);
  1458. found_passing_read = 0;
  1459. found_failing_read = 0;
  1460. initial_failing_dtap = d;
  1461. for (; d <= IO_DQS_EN_DELAY_MAX; d++) {
  1462. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: testing \
  1463. read d=%u\n", __func__, __LINE__, d);
  1464. scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
  1465. if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
  1466. PASS_ONE_BIT,
  1467. &bit_chk, 0)) {
  1468. found_passing_read = 1;
  1469. break;
  1470. }
  1471. }
  1472. if (found_passing_read) {
  1473. /* Find a failing read */
  1474. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find failing \
  1475. read\n", __func__, __LINE__);
  1476. for (d = d + 1; d <= IO_DQS_EN_DELAY_MAX; d++) {
  1477. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \
  1478. testing read d=%u\n", __func__, __LINE__, d);
  1479. scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
  1480. if (!rw_mgr_mem_calibrate_read_test_all_ranks
  1481. (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) {
  1482. found_failing_read = 1;
  1483. break;
  1484. }
  1485. }
  1486. } else {
  1487. debug_cond(DLEVEL == 1, "%s:%d find_dqs_en_phase: failed to \
  1488. calculate dtaps", __func__, __LINE__);
  1489. debug_cond(DLEVEL == 1, "per ptap. Fall back on static value\n");
  1490. }
  1491. /*
  1492. * The dynamically calculated dtaps_per_ptap is only valid if we
  1493. * found a passing/failing read. If we didn't, it means d hit the max
  1494. * (IO_DQS_EN_DELAY_MAX). Otherwise, dtaps_per_ptap retains its
  1495. * statically calculated value.
  1496. */
  1497. if (found_passing_read && found_failing_read)
  1498. dtaps_per_ptap = d - initial_failing_dtap;
  1499. writel(dtaps_per_ptap, &sdr_reg_file->dtaps_per_ptap);
  1500. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: dtaps_per_ptap=%u \
  1501. - %u = %u", __func__, __LINE__, d,
  1502. initial_failing_dtap, dtaps_per_ptap);
  1503. /* ******************************************** */
  1504. /* * step 6: Find the centre of the window * */
  1505. if (sdr_find_window_centre(grp, work_bgn, work_end, v))
  1506. return 0; /* FIXME: Old code, return 0 means failure :-( */
  1507. return 1;
  1508. }
  1509. /* per-bit deskew DQ and center */
  1510. static uint32_t rw_mgr_mem_calibrate_vfifo_center(uint32_t rank_bgn,
  1511. uint32_t write_group, uint32_t read_group, uint32_t test_bgn,
  1512. uint32_t use_read_test, uint32_t update_fom)
  1513. {
  1514. uint32_t i, p, d, min_index;
  1515. /*
  1516. * Store these as signed since there are comparisons with
  1517. * signed numbers.
  1518. */
  1519. uint32_t bit_chk;
  1520. uint32_t sticky_bit_chk;
  1521. int32_t left_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
  1522. int32_t right_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
  1523. int32_t final_dq[RW_MGR_MEM_DQ_PER_READ_DQS];
  1524. int32_t mid;
  1525. int32_t orig_mid_min, mid_min;
  1526. int32_t new_dqs, start_dqs, start_dqs_en, shift_dq, final_dqs,
  1527. final_dqs_en;
  1528. int32_t dq_margin, dqs_margin;
  1529. uint32_t stop;
  1530. uint32_t temp_dq_in_delay1, temp_dq_in_delay2;
  1531. uint32_t addr;
  1532. debug("%s:%d: %u %u", __func__, __LINE__, read_group, test_bgn);
  1533. addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_DQS_IN_DELAY_OFFSET;
  1534. start_dqs = readl(addr + (read_group << 2));
  1535. if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
  1536. start_dqs_en = readl(addr + ((read_group << 2)
  1537. - IO_DQS_EN_DELAY_OFFSET));
  1538. /* set the left and right edge of each bit to an illegal value */
  1539. /* use (IO_IO_IN_DELAY_MAX + 1) as an illegal value */
  1540. sticky_bit_chk = 0;
  1541. for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
  1542. left_edge[i] = IO_IO_IN_DELAY_MAX + 1;
  1543. right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
  1544. }
  1545. /* Search for the left edge of the window for each bit */
  1546. for (d = 0; d <= IO_IO_IN_DELAY_MAX; d++) {
  1547. scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, d);
  1548. writel(0, &sdr_scc_mgr->update);
  1549. /*
  1550. * Stop searching when the read test doesn't pass AND when
  1551. * we've seen a passing read on every bit.
  1552. */
  1553. if (use_read_test) {
  1554. stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
  1555. read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
  1556. &bit_chk, 0, 0);
  1557. } else {
  1558. rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
  1559. 0, PASS_ONE_BIT,
  1560. &bit_chk, 0);
  1561. bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
  1562. (read_group - (write_group *
  1563. RW_MGR_MEM_IF_READ_DQS_WIDTH /
  1564. RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
  1565. stop = (bit_chk == 0);
  1566. }
  1567. sticky_bit_chk = sticky_bit_chk | bit_chk;
  1568. stop = stop && (sticky_bit_chk == param->read_correct_mask);
  1569. debug_cond(DLEVEL == 2, "%s:%d vfifo_center(left): dtap=%u => %u == %u \
  1570. && %u", __func__, __LINE__, d,
  1571. sticky_bit_chk,
  1572. param->read_correct_mask, stop);
  1573. if (stop == 1) {
  1574. break;
  1575. } else {
  1576. for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
  1577. if (bit_chk & 1) {
  1578. /* Remember a passing test as the
  1579. left_edge */
  1580. left_edge[i] = d;
  1581. } else {
  1582. /* If a left edge has not been seen yet,
  1583. then a future passing test will mark
  1584. this edge as the right edge */
  1585. if (left_edge[i] ==
  1586. IO_IO_IN_DELAY_MAX + 1) {
  1587. right_edge[i] = -(d + 1);
  1588. }
  1589. }
  1590. bit_chk = bit_chk >> 1;
  1591. }
  1592. }
  1593. }
  1594. /* Reset DQ delay chains to 0 */
  1595. scc_mgr_apply_group_dq_in_delay(test_bgn, 0);
  1596. sticky_bit_chk = 0;
  1597. for (i = RW_MGR_MEM_DQ_PER_READ_DQS - 1;; i--) {
  1598. debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
  1599. %d right_edge[%u]: %d\n", __func__, __LINE__,
  1600. i, left_edge[i], i, right_edge[i]);
  1601. /*
  1602. * Check for cases where we haven't found the left edge,
  1603. * which makes our assignment of the the right edge invalid.
  1604. * Reset it to the illegal value.
  1605. */
  1606. if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) && (
  1607. right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
  1608. right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
  1609. debug_cond(DLEVEL == 2, "%s:%d vfifo_center: reset \
  1610. right_edge[%u]: %d\n", __func__, __LINE__,
  1611. i, right_edge[i]);
  1612. }
  1613. /*
  1614. * Reset sticky bit (except for bits where we have seen
  1615. * both the left and right edge).
  1616. */
  1617. sticky_bit_chk = sticky_bit_chk << 1;
  1618. if ((left_edge[i] != IO_IO_IN_DELAY_MAX + 1) &&
  1619. (right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
  1620. sticky_bit_chk = sticky_bit_chk | 1;
  1621. }
  1622. if (i == 0)
  1623. break;
  1624. }
  1625. /* Search for the right edge of the window for each bit */
  1626. for (d = 0; d <= IO_DQS_IN_DELAY_MAX - start_dqs; d++) {
  1627. scc_mgr_set_dqs_bus_in_delay(read_group, d + start_dqs);
  1628. if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
  1629. uint32_t delay = d + start_dqs_en;
  1630. if (delay > IO_DQS_EN_DELAY_MAX)
  1631. delay = IO_DQS_EN_DELAY_MAX;
  1632. scc_mgr_set_dqs_en_delay(read_group, delay);
  1633. }
  1634. scc_mgr_load_dqs(read_group);
  1635. writel(0, &sdr_scc_mgr->update);
  1636. /*
  1637. * Stop searching when the read test doesn't pass AND when
  1638. * we've seen a passing read on every bit.
  1639. */
  1640. if (use_read_test) {
  1641. stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
  1642. read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
  1643. &bit_chk, 0, 0);
  1644. } else {
  1645. rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
  1646. 0, PASS_ONE_BIT,
  1647. &bit_chk, 0);
  1648. bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
  1649. (read_group - (write_group *
  1650. RW_MGR_MEM_IF_READ_DQS_WIDTH /
  1651. RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
  1652. stop = (bit_chk == 0);
  1653. }
  1654. sticky_bit_chk = sticky_bit_chk | bit_chk;
  1655. stop = stop && (sticky_bit_chk == param->read_correct_mask);
  1656. debug_cond(DLEVEL == 2, "%s:%d vfifo_center(right): dtap=%u => %u == \
  1657. %u && %u", __func__, __LINE__, d,
  1658. sticky_bit_chk, param->read_correct_mask, stop);
  1659. if (stop == 1) {
  1660. break;
  1661. } else {
  1662. for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
  1663. if (bit_chk & 1) {
  1664. /* Remember a passing test as
  1665. the right_edge */
  1666. right_edge[i] = d;
  1667. } else {
  1668. if (d != 0) {
  1669. /* If a right edge has not been
  1670. seen yet, then a future passing
  1671. test will mark this edge as the
  1672. left edge */
  1673. if (right_edge[i] ==
  1674. IO_IO_IN_DELAY_MAX + 1) {
  1675. left_edge[i] = -(d + 1);
  1676. }
  1677. } else {
  1678. /* d = 0 failed, but it passed
  1679. when testing the left edge,
  1680. so it must be marginal,
  1681. set it to -1 */
  1682. if (right_edge[i] ==
  1683. IO_IO_IN_DELAY_MAX + 1 &&
  1684. left_edge[i] !=
  1685. IO_IO_IN_DELAY_MAX
  1686. + 1) {
  1687. right_edge[i] = -1;
  1688. }
  1689. /* If a right edge has not been
  1690. seen yet, then a future passing
  1691. test will mark this edge as the
  1692. left edge */
  1693. else if (right_edge[i] ==
  1694. IO_IO_IN_DELAY_MAX +
  1695. 1) {
  1696. left_edge[i] = -(d + 1);
  1697. }
  1698. }
  1699. }
  1700. debug_cond(DLEVEL == 2, "%s:%d vfifo_center[r,\
  1701. d=%u]: ", __func__, __LINE__, d);
  1702. debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d ",
  1703. (int)(bit_chk & 1), i, left_edge[i]);
  1704. debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
  1705. right_edge[i]);
  1706. bit_chk = bit_chk >> 1;
  1707. }
  1708. }
  1709. }
  1710. /* Check that all bits have a window */
  1711. for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
  1712. debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
  1713. %d right_edge[%u]: %d", __func__, __LINE__,
  1714. i, left_edge[i], i, right_edge[i]);
  1715. if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) || (right_edge[i]
  1716. == IO_IO_IN_DELAY_MAX + 1)) {
  1717. /*
  1718. * Restore delay chain settings before letting the loop
  1719. * in rw_mgr_mem_calibrate_vfifo to retry different
  1720. * dqs/ck relationships.
  1721. */
  1722. scc_mgr_set_dqs_bus_in_delay(read_group, start_dqs);
  1723. if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
  1724. scc_mgr_set_dqs_en_delay(read_group,
  1725. start_dqs_en);
  1726. }
  1727. scc_mgr_load_dqs(read_group);
  1728. writel(0, &sdr_scc_mgr->update);
  1729. debug_cond(DLEVEL == 1, "%s:%d vfifo_center: failed to \
  1730. find edge [%u]: %d %d", __func__, __LINE__,
  1731. i, left_edge[i], right_edge[i]);
  1732. if (use_read_test) {
  1733. set_failing_group_stage(read_group *
  1734. RW_MGR_MEM_DQ_PER_READ_DQS + i,
  1735. CAL_STAGE_VFIFO,
  1736. CAL_SUBSTAGE_VFIFO_CENTER);
  1737. } else {
  1738. set_failing_group_stage(read_group *
  1739. RW_MGR_MEM_DQ_PER_READ_DQS + i,
  1740. CAL_STAGE_VFIFO_AFTER_WRITES,
  1741. CAL_SUBSTAGE_VFIFO_CENTER);
  1742. }
  1743. return 0;
  1744. }
  1745. }
  1746. /* Find middle of window for each DQ bit */
  1747. mid_min = left_edge[0] - right_edge[0];
  1748. min_index = 0;
  1749. for (i = 1; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
  1750. mid = left_edge[i] - right_edge[i];
  1751. if (mid < mid_min) {
  1752. mid_min = mid;
  1753. min_index = i;
  1754. }
  1755. }
  1756. /*
  1757. * -mid_min/2 represents the amount that we need to move DQS.
  1758. * If mid_min is odd and positive we'll need to add one to
  1759. * make sure the rounding in further calculations is correct
  1760. * (always bias to the right), so just add 1 for all positive values.
  1761. */
  1762. if (mid_min > 0)
  1763. mid_min++;
  1764. mid_min = mid_min / 2;
  1765. debug_cond(DLEVEL == 1, "%s:%d vfifo_center: mid_min=%d (index=%u)\n",
  1766. __func__, __LINE__, mid_min, min_index);
  1767. /* Determine the amount we can change DQS (which is -mid_min) */
  1768. orig_mid_min = mid_min;
  1769. new_dqs = start_dqs - mid_min;
  1770. if (new_dqs > IO_DQS_IN_DELAY_MAX)
  1771. new_dqs = IO_DQS_IN_DELAY_MAX;
  1772. else if (new_dqs < 0)
  1773. new_dqs = 0;
  1774. mid_min = start_dqs - new_dqs;
  1775. debug_cond(DLEVEL == 1, "vfifo_center: new mid_min=%d new_dqs=%d\n",
  1776. mid_min, new_dqs);
  1777. if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
  1778. if (start_dqs_en - mid_min > IO_DQS_EN_DELAY_MAX)
  1779. mid_min += start_dqs_en - mid_min - IO_DQS_EN_DELAY_MAX;
  1780. else if (start_dqs_en - mid_min < 0)
  1781. mid_min += start_dqs_en - mid_min;
  1782. }
  1783. new_dqs = start_dqs - mid_min;
  1784. debug_cond(DLEVEL == 1, "vfifo_center: start_dqs=%d start_dqs_en=%d \
  1785. new_dqs=%d mid_min=%d\n", start_dqs,
  1786. IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS ? start_dqs_en : -1,
  1787. new_dqs, mid_min);
  1788. /* Initialize data for export structures */
  1789. dqs_margin = IO_IO_IN_DELAY_MAX + 1;
  1790. dq_margin = IO_IO_IN_DELAY_MAX + 1;
  1791. /* add delay to bring centre of all DQ windows to the same "level" */
  1792. for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
  1793. /* Use values before divide by 2 to reduce round off error */
  1794. shift_dq = (left_edge[i] - right_edge[i] -
  1795. (left_edge[min_index] - right_edge[min_index]))/2 +
  1796. (orig_mid_min - mid_min);
  1797. debug_cond(DLEVEL == 2, "vfifo_center: before: \
  1798. shift_dq[%u]=%d\n", i, shift_dq);
  1799. addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_IN_DELAY_OFFSET;
  1800. temp_dq_in_delay1 = readl(addr + (p << 2));
  1801. temp_dq_in_delay2 = readl(addr + (i << 2));
  1802. if (shift_dq + (int32_t)temp_dq_in_delay1 >
  1803. (int32_t)IO_IO_IN_DELAY_MAX) {
  1804. shift_dq = (int32_t)IO_IO_IN_DELAY_MAX - temp_dq_in_delay2;
  1805. } else if (shift_dq + (int32_t)temp_dq_in_delay1 < 0) {
  1806. shift_dq = -(int32_t)temp_dq_in_delay1;
  1807. }
  1808. debug_cond(DLEVEL == 2, "vfifo_center: after: \
  1809. shift_dq[%u]=%d\n", i, shift_dq);
  1810. final_dq[i] = temp_dq_in_delay1 + shift_dq;
  1811. scc_mgr_set_dq_in_delay(p, final_dq[i]);
  1812. scc_mgr_load_dq(p);
  1813. debug_cond(DLEVEL == 2, "vfifo_center: margin[%u]=[%d,%d]\n", i,
  1814. left_edge[i] - shift_dq + (-mid_min),
  1815. right_edge[i] + shift_dq - (-mid_min));
  1816. /* To determine values for export structures */
  1817. if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
  1818. dq_margin = left_edge[i] - shift_dq + (-mid_min);
  1819. if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
  1820. dqs_margin = right_edge[i] + shift_dq - (-mid_min);
  1821. }
  1822. final_dqs = new_dqs;
  1823. if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
  1824. final_dqs_en = start_dqs_en - mid_min;
  1825. /* Move DQS-en */
  1826. if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
  1827. scc_mgr_set_dqs_en_delay(read_group, final_dqs_en);
  1828. scc_mgr_load_dqs(read_group);
  1829. }
  1830. /* Move DQS */
  1831. scc_mgr_set_dqs_bus_in_delay(read_group, final_dqs);
  1832. scc_mgr_load_dqs(read_group);
  1833. debug_cond(DLEVEL == 2, "%s:%d vfifo_center: dq_margin=%d \
  1834. dqs_margin=%d", __func__, __LINE__,
  1835. dq_margin, dqs_margin);
  1836. /*
  1837. * Do not remove this line as it makes sure all of our decisions
  1838. * have been applied. Apply the update bit.
  1839. */
  1840. writel(0, &sdr_scc_mgr->update);
  1841. return (dq_margin >= 0) && (dqs_margin >= 0);
  1842. }
  1843. /**
  1844. * rw_mgr_mem_calibrate_guaranteed_write() - Perform guaranteed write into the device
  1845. * @rw_group: Read/Write Group
  1846. * @phase: DQ/DQS phase
  1847. *
  1848. * Because initially no communication ca be reliably performed with the memory
  1849. * device, the sequencer uses a guaranteed write mechanism to write data into
  1850. * the memory device.
  1851. */
  1852. static int rw_mgr_mem_calibrate_guaranteed_write(const u32 rw_group,
  1853. const u32 phase)
  1854. {
  1855. int ret;
  1856. /* Set a particular DQ/DQS phase. */
  1857. scc_mgr_set_dqdqs_output_phase_all_ranks(rw_group, phase);
  1858. debug_cond(DLEVEL == 1, "%s:%d guaranteed write: g=%u p=%u\n",
  1859. __func__, __LINE__, rw_group, phase);
  1860. /*
  1861. * Altera EMI_RM 2015.05.04 :: Figure 1-25
  1862. * Load up the patterns used by read calibration using the
  1863. * current DQDQS phase.
  1864. */
  1865. rw_mgr_mem_calibrate_read_load_patterns(0, 1);
  1866. if (gbl->phy_debug_mode_flags & PHY_DEBUG_DISABLE_GUARANTEED_READ)
  1867. return 0;
  1868. /*
  1869. * Altera EMI_RM 2015.05.04 :: Figure 1-26
  1870. * Back-to-Back reads of the patterns used for calibration.
  1871. */
  1872. ret = rw_mgr_mem_calibrate_read_test_patterns(0, rw_group, 1);
  1873. if (ret)
  1874. debug_cond(DLEVEL == 1,
  1875. "%s:%d Guaranteed read test failed: g=%u p=%u\n",
  1876. __func__, __LINE__, rw_group, phase);
  1877. return ret;
  1878. }
  1879. /**
  1880. * rw_mgr_mem_calibrate_dqs_enable_calibration() - DQS Enable Calibration
  1881. * @rw_group: Read/Write Group
  1882. * @test_bgn: Rank at which the test begins
  1883. *
  1884. * DQS enable calibration ensures reliable capture of the DQ signal without
  1885. * glitches on the DQS line.
  1886. */
  1887. static int rw_mgr_mem_calibrate_dqs_enable_calibration(const u32 rw_group,
  1888. const u32 test_bgn)
  1889. {
  1890. /*
  1891. * Altera EMI_RM 2015.05.04 :: Figure 1-27
  1892. * DQS and DQS Eanble Signal Relationships.
  1893. */
  1894. /* We start at zero, so have one less dq to devide among */
  1895. const u32 delay_step = IO_IO_IN_DELAY_MAX /
  1896. (RW_MGR_MEM_DQ_PER_READ_DQS - 1);
  1897. int found;
  1898. u32 i, p, d, r;
  1899. debug("%s:%d (%u,%u)\n", __func__, __LINE__, rw_group, test_bgn);
  1900. /* Try different dq_in_delays since the DQ path is shorter than DQS. */
  1901. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
  1902. r += NUM_RANKS_PER_SHADOW_REG) {
  1903. for (i = 0, p = test_bgn, d = 0;
  1904. i < RW_MGR_MEM_DQ_PER_READ_DQS;
  1905. i++, p++, d += delay_step) {
  1906. debug_cond(DLEVEL == 1,
  1907. "%s:%d: g=%u r=%u i=%u p=%u d=%u\n",
  1908. __func__, __LINE__, rw_group, r, i, p, d);
  1909. scc_mgr_set_dq_in_delay(p, d);
  1910. scc_mgr_load_dq(p);
  1911. }
  1912. writel(0, &sdr_scc_mgr->update);
  1913. }
  1914. /*
  1915. * Try rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase across different
  1916. * dq_in_delay values
  1917. */
  1918. found = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(rw_group);
  1919. debug_cond(DLEVEL == 1,
  1920. "%s:%d: g=%u found=%u; Reseting delay chain to zero\n",
  1921. __func__, __LINE__, rw_group, found);
  1922. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
  1923. r += NUM_RANKS_PER_SHADOW_REG) {
  1924. scc_mgr_apply_group_dq_in_delay(test_bgn, 0);
  1925. writel(0, &sdr_scc_mgr->update);
  1926. }
  1927. if (!found)
  1928. return -EINVAL;
  1929. return 0;
  1930. }
  1931. /**
  1932. * rw_mgr_mem_calibrate_dq_dqs_centering() - Centering DQ/DQS
  1933. * @rw_group: Read/Write Group
  1934. * @test_bgn: Rank at which the test begins
  1935. * @use_read_test: Perform a read test
  1936. * @update_fom: Update FOM
  1937. *
  1938. * The centerin DQ/DQS stage attempts to align DQ and DQS signals on reads
  1939. * within a group.
  1940. */
  1941. static int
  1942. rw_mgr_mem_calibrate_dq_dqs_centering(const u32 rw_group, const u32 test_bgn,
  1943. const int use_read_test,
  1944. const int update_fom)
  1945. {
  1946. int ret, grp_calibrated;
  1947. u32 rank_bgn, sr;
  1948. /*
  1949. * Altera EMI_RM 2015.05.04 :: Figure 1-28
  1950. * Read per-bit deskew can be done on a per shadow register basis.
  1951. */
  1952. grp_calibrated = 1;
  1953. for (rank_bgn = 0, sr = 0;
  1954. rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
  1955. rank_bgn += NUM_RANKS_PER_SHADOW_REG, sr++) {
  1956. /* Check if this set of ranks should be skipped entirely. */
  1957. if (param->skip_shadow_regs[sr])
  1958. continue;
  1959. ret = rw_mgr_mem_calibrate_vfifo_center(rank_bgn, rw_group,
  1960. rw_group, test_bgn,
  1961. use_read_test,
  1962. update_fom);
  1963. if (ret)
  1964. continue;
  1965. grp_calibrated = 0;
  1966. }
  1967. if (!grp_calibrated)
  1968. return -EIO;
  1969. return 0;
  1970. }
  1971. /**
  1972. * rw_mgr_mem_calibrate_vfifo() - Calibrate the read valid prediction FIFO
  1973. * @rw_group: Read/Write Group
  1974. * @test_bgn: Rank at which the test begins
  1975. *
  1976. * Stage 1: Calibrate the read valid prediction FIFO.
  1977. *
  1978. * This function implements UniPHY calibration Stage 1, as explained in
  1979. * detail in Altera EMI_RM 2015.05.04 , "UniPHY Calibration Stages".
  1980. *
  1981. * - read valid prediction will consist of finding:
  1982. * - DQS enable phase and DQS enable delay (DQS Enable Calibration)
  1983. * - DQS input phase and DQS input delay (DQ/DQS Centering)
  1984. * - we also do a per-bit deskew on the DQ lines.
  1985. */
  1986. static int rw_mgr_mem_calibrate_vfifo(const u32 rw_group, const u32 test_bgn)
  1987. {
  1988. uint32_t p, d;
  1989. uint32_t dtaps_per_ptap;
  1990. uint32_t failed_substage;
  1991. int ret;
  1992. debug("%s:%d: %u %u\n", __func__, __LINE__, rw_group, test_bgn);
  1993. /* Update info for sims */
  1994. reg_file_set_group(rw_group);
  1995. reg_file_set_stage(CAL_STAGE_VFIFO);
  1996. reg_file_set_sub_stage(CAL_SUBSTAGE_GUARANTEED_READ);
  1997. failed_substage = CAL_SUBSTAGE_GUARANTEED_READ;
  1998. /* USER Determine number of delay taps for each phase tap. */
  1999. dtaps_per_ptap = DIV_ROUND_UP(IO_DELAY_PER_OPA_TAP,
  2000. IO_DELAY_PER_DQS_EN_DCHAIN_TAP) - 1;
  2001. for (d = 0; d <= dtaps_per_ptap; d += 2) {
  2002. /*
  2003. * In RLDRAMX we may be messing the delay of pins in
  2004. * the same write rw_group but outside of the current read
  2005. * the rw_group, but that's ok because we haven't calibrated
  2006. * output side yet.
  2007. */
  2008. if (d > 0) {
  2009. scc_mgr_apply_group_all_out_delay_add_all_ranks(
  2010. rw_group, d);
  2011. }
  2012. for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX; p++) {
  2013. /* 1) Guaranteed Write */
  2014. ret = rw_mgr_mem_calibrate_guaranteed_write(rw_group, p);
  2015. if (ret)
  2016. break;
  2017. /* 2) DQS Enable Calibration */
  2018. ret = rw_mgr_mem_calibrate_dqs_enable_calibration(rw_group,
  2019. test_bgn);
  2020. if (ret) {
  2021. failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE;
  2022. continue;
  2023. }
  2024. /* 3) Centering DQ/DQS */
  2025. /*
  2026. * If doing read after write calibration, do not update
  2027. * FOM now. Do it then.
  2028. */
  2029. ret = rw_mgr_mem_calibrate_dq_dqs_centering(rw_group,
  2030. test_bgn, 1, 0);
  2031. if (ret) {
  2032. failed_substage = CAL_SUBSTAGE_VFIFO_CENTER;
  2033. continue;
  2034. }
  2035. /* All done. */
  2036. goto cal_done_ok;
  2037. }
  2038. }
  2039. /* Calibration Stage 1 failed. */
  2040. set_failing_group_stage(rw_group, CAL_STAGE_VFIFO, failed_substage);
  2041. return 0;
  2042. /* Calibration Stage 1 completed OK. */
  2043. cal_done_ok:
  2044. /*
  2045. * Reset the delay chains back to zero if they have moved > 1
  2046. * (check for > 1 because loop will increase d even when pass in
  2047. * first case).
  2048. */
  2049. if (d > 2)
  2050. scc_mgr_zero_group(rw_group, 1);
  2051. return 1;
  2052. }
  2053. /* VFIFO Calibration -- Read Deskew Calibration after write deskew */
  2054. static uint32_t rw_mgr_mem_calibrate_vfifo_end(uint32_t read_group,
  2055. uint32_t test_bgn)
  2056. {
  2057. uint32_t rank_bgn, sr;
  2058. uint32_t grp_calibrated;
  2059. uint32_t write_group;
  2060. debug("%s:%d %u %u", __func__, __LINE__, read_group, test_bgn);
  2061. /* update info for sims */
  2062. reg_file_set_stage(CAL_STAGE_VFIFO_AFTER_WRITES);
  2063. reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
  2064. write_group = read_group;
  2065. /* update info for sims */
  2066. reg_file_set_group(read_group);
  2067. grp_calibrated = 1;
  2068. /* Read per-bit deskew can be done on a per shadow register basis */
  2069. for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
  2070. rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) {
  2071. /* Determine if this set of ranks should be skipped entirely */
  2072. if (!param->skip_shadow_regs[sr]) {
  2073. /* This is the last calibration round, update FOM here */
  2074. if (!rw_mgr_mem_calibrate_vfifo_center(rank_bgn,
  2075. write_group,
  2076. read_group,
  2077. test_bgn, 0,
  2078. 1)) {
  2079. grp_calibrated = 0;
  2080. }
  2081. }
  2082. }
  2083. if (grp_calibrated == 0) {
  2084. set_failing_group_stage(write_group,
  2085. CAL_STAGE_VFIFO_AFTER_WRITES,
  2086. CAL_SUBSTAGE_VFIFO_CENTER);
  2087. return 0;
  2088. }
  2089. return 1;
  2090. }
  2091. /* Calibrate LFIFO to find smallest read latency */
  2092. static uint32_t rw_mgr_mem_calibrate_lfifo(void)
  2093. {
  2094. uint32_t found_one;
  2095. uint32_t bit_chk;
  2096. debug("%s:%d\n", __func__, __LINE__);
  2097. /* update info for sims */
  2098. reg_file_set_stage(CAL_STAGE_LFIFO);
  2099. reg_file_set_sub_stage(CAL_SUBSTAGE_READ_LATENCY);
  2100. /* Load up the patterns used by read calibration for all ranks */
  2101. rw_mgr_mem_calibrate_read_load_patterns(0, 1);
  2102. found_one = 0;
  2103. do {
  2104. writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
  2105. debug_cond(DLEVEL == 2, "%s:%d lfifo: read_lat=%u",
  2106. __func__, __LINE__, gbl->curr_read_lat);
  2107. if (!rw_mgr_mem_calibrate_read_test_all_ranks(0,
  2108. NUM_READ_TESTS,
  2109. PASS_ALL_BITS,
  2110. &bit_chk, 1)) {
  2111. break;
  2112. }
  2113. found_one = 1;
  2114. /* reduce read latency and see if things are working */
  2115. /* correctly */
  2116. gbl->curr_read_lat--;
  2117. } while (gbl->curr_read_lat > 0);
  2118. /* reset the fifos to get pointers to known state */
  2119. writel(0, &phy_mgr_cmd->fifo_reset);
  2120. if (found_one) {
  2121. /* add a fudge factor to the read latency that was determined */
  2122. gbl->curr_read_lat += 2;
  2123. writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
  2124. debug_cond(DLEVEL == 2, "%s:%d lfifo: success: using \
  2125. read_lat=%u\n", __func__, __LINE__,
  2126. gbl->curr_read_lat);
  2127. return 1;
  2128. } else {
  2129. set_failing_group_stage(0xff, CAL_STAGE_LFIFO,
  2130. CAL_SUBSTAGE_READ_LATENCY);
  2131. debug_cond(DLEVEL == 2, "%s:%d lfifo: failed at initial \
  2132. read_lat=%u\n", __func__, __LINE__,
  2133. gbl->curr_read_lat);
  2134. return 0;
  2135. }
  2136. }
  2137. /*
  2138. * issue write test command.
  2139. * two variants are provided. one that just tests a write pattern and
  2140. * another that tests datamask functionality.
  2141. */
  2142. static void rw_mgr_mem_calibrate_write_test_issue(uint32_t group,
  2143. uint32_t test_dm)
  2144. {
  2145. uint32_t mcc_instruction;
  2146. uint32_t quick_write_mode = (((STATIC_CALIB_STEPS) & CALIB_SKIP_WRITES) &&
  2147. ENABLE_SUPER_QUICK_CALIBRATION);
  2148. uint32_t rw_wl_nop_cycles;
  2149. uint32_t addr;
  2150. /*
  2151. * Set counter and jump addresses for the right
  2152. * number of NOP cycles.
  2153. * The number of supported NOP cycles can range from -1 to infinity
  2154. * Three different cases are handled:
  2155. *
  2156. * 1. For a number of NOP cycles greater than 0, the RW Mgr looping
  2157. * mechanism will be used to insert the right number of NOPs
  2158. *
  2159. * 2. For a number of NOP cycles equals to 0, the micro-instruction
  2160. * issuing the write command will jump straight to the
  2161. * micro-instruction that turns on DQS (for DDRx), or outputs write
  2162. * data (for RLD), skipping
  2163. * the NOP micro-instruction all together
  2164. *
  2165. * 3. A number of NOP cycles equal to -1 indicates that DQS must be
  2166. * turned on in the same micro-instruction that issues the write
  2167. * command. Then we need
  2168. * to directly jump to the micro-instruction that sends out the data
  2169. *
  2170. * NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters
  2171. * (2 and 3). One jump-counter (0) is used to perform multiple
  2172. * write-read operations.
  2173. * one counter left to issue this command in "multiple-group" mode
  2174. */
  2175. rw_wl_nop_cycles = gbl->rw_wl_nop_cycles;
  2176. if (rw_wl_nop_cycles == -1) {
  2177. /*
  2178. * CNTR 2 - We want to execute the special write operation that
  2179. * turns on DQS right away and then skip directly to the
  2180. * instruction that sends out the data. We set the counter to a
  2181. * large number so that the jump is always taken.
  2182. */
  2183. writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
  2184. /* CNTR 3 - Not used */
  2185. if (test_dm) {
  2186. mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0_WL_1;
  2187. writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DATA,
  2188. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  2189. writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP,
  2190. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  2191. } else {
  2192. mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0_WL_1;
  2193. writel(RW_MGR_LFSR_WR_RD_BANK_0_DATA,
  2194. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  2195. writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP,
  2196. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  2197. }
  2198. } else if (rw_wl_nop_cycles == 0) {
  2199. /*
  2200. * CNTR 2 - We want to skip the NOP operation and go straight
  2201. * to the DQS enable instruction. We set the counter to a large
  2202. * number so that the jump is always taken.
  2203. */
  2204. writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
  2205. /* CNTR 3 - Not used */
  2206. if (test_dm) {
  2207. mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
  2208. writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DQS,
  2209. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  2210. } else {
  2211. mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
  2212. writel(RW_MGR_LFSR_WR_RD_BANK_0_DQS,
  2213. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  2214. }
  2215. } else {
  2216. /*
  2217. * CNTR 2 - In this case we want to execute the next instruction
  2218. * and NOT take the jump. So we set the counter to 0. The jump
  2219. * address doesn't count.
  2220. */
  2221. writel(0x0, &sdr_rw_load_mgr_regs->load_cntr2);
  2222. writel(0x0, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  2223. /*
  2224. * CNTR 3 - Set the nop counter to the number of cycles we
  2225. * need to loop for, minus 1.
  2226. */
  2227. writel(rw_wl_nop_cycles - 1, &sdr_rw_load_mgr_regs->load_cntr3);
  2228. if (test_dm) {
  2229. mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
  2230. writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP,
  2231. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  2232. } else {
  2233. mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
  2234. writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP,
  2235. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  2236. }
  2237. }
  2238. writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  2239. RW_MGR_RESET_READ_DATAPATH_OFFSET);
  2240. if (quick_write_mode)
  2241. writel(0x08, &sdr_rw_load_mgr_regs->load_cntr0);
  2242. else
  2243. writel(0x40, &sdr_rw_load_mgr_regs->load_cntr0);
  2244. writel(mcc_instruction, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  2245. /*
  2246. * CNTR 1 - This is used to ensure enough time elapses
  2247. * for read data to come back.
  2248. */
  2249. writel(0x30, &sdr_rw_load_mgr_regs->load_cntr1);
  2250. if (test_dm) {
  2251. writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_WAIT,
  2252. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  2253. } else {
  2254. writel(RW_MGR_LFSR_WR_RD_BANK_0_WAIT,
  2255. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  2256. }
  2257. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  2258. writel(mcc_instruction, addr + (group << 2));
  2259. }
  2260. /* Test writes, can check for a single bit pass or multiple bit pass */
  2261. static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
  2262. uint32_t write_group, uint32_t use_dm, uint32_t all_correct,
  2263. uint32_t *bit_chk, uint32_t all_ranks)
  2264. {
  2265. uint32_t r;
  2266. uint32_t correct_mask_vg;
  2267. uint32_t tmp_bit_chk;
  2268. uint32_t vg;
  2269. uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
  2270. (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
  2271. uint32_t addr_rw_mgr;
  2272. uint32_t base_rw_mgr;
  2273. *bit_chk = param->write_correct_mask;
  2274. correct_mask_vg = param->write_correct_mask_vg;
  2275. for (r = rank_bgn; r < rank_end; r++) {
  2276. if (param->skip_ranks[r]) {
  2277. /* request to skip the rank */
  2278. continue;
  2279. }
  2280. /* set rank */
  2281. set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
  2282. tmp_bit_chk = 0;
  2283. addr_rw_mgr = SDR_PHYGRP_RWMGRGRP_ADDRESS;
  2284. for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS-1; ; vg--) {
  2285. /* reset the fifos to get pointers to known state */
  2286. writel(0, &phy_mgr_cmd->fifo_reset);
  2287. tmp_bit_chk = tmp_bit_chk <<
  2288. (RW_MGR_MEM_DQ_PER_WRITE_DQS /
  2289. RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
  2290. rw_mgr_mem_calibrate_write_test_issue(write_group *
  2291. RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS+vg,
  2292. use_dm);
  2293. base_rw_mgr = readl(addr_rw_mgr);
  2294. tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
  2295. if (vg == 0)
  2296. break;
  2297. }
  2298. *bit_chk &= tmp_bit_chk;
  2299. }
  2300. if (all_correct) {
  2301. set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
  2302. debug_cond(DLEVEL == 2, "write_test(%u,%u,ALL) : %u == \
  2303. %u => %lu", write_group, use_dm,
  2304. *bit_chk, param->write_correct_mask,
  2305. (long unsigned int)(*bit_chk ==
  2306. param->write_correct_mask));
  2307. return *bit_chk == param->write_correct_mask;
  2308. } else {
  2309. set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
  2310. debug_cond(DLEVEL == 2, "write_test(%u,%u,ONE) : %u != ",
  2311. write_group, use_dm, *bit_chk);
  2312. debug_cond(DLEVEL == 2, "%lu" " => %lu", (long unsigned int)0,
  2313. (long unsigned int)(*bit_chk != 0));
  2314. return *bit_chk != 0x00;
  2315. }
  2316. }
  2317. /*
  2318. * center all windows. do per-bit-deskew to possibly increase size of
  2319. * certain windows.
  2320. */
  2321. static uint32_t rw_mgr_mem_calibrate_writes_center(uint32_t rank_bgn,
  2322. uint32_t write_group, uint32_t test_bgn)
  2323. {
  2324. uint32_t i, p, min_index;
  2325. int32_t d;
  2326. /*
  2327. * Store these as signed since there are comparisons with
  2328. * signed numbers.
  2329. */
  2330. uint32_t bit_chk;
  2331. uint32_t sticky_bit_chk;
  2332. int32_t left_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
  2333. int32_t right_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
  2334. int32_t mid;
  2335. int32_t mid_min, orig_mid_min;
  2336. int32_t new_dqs, start_dqs, shift_dq;
  2337. int32_t dq_margin, dqs_margin, dm_margin;
  2338. uint32_t stop;
  2339. uint32_t temp_dq_out1_delay;
  2340. uint32_t addr;
  2341. debug("%s:%d %u %u", __func__, __LINE__, write_group, test_bgn);
  2342. dm_margin = 0;
  2343. addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
  2344. start_dqs = readl(addr +
  2345. (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2));
  2346. /* per-bit deskew */
  2347. /*
  2348. * set the left and right edge of each bit to an illegal value
  2349. * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value.
  2350. */
  2351. sticky_bit_chk = 0;
  2352. for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
  2353. left_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
  2354. right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
  2355. }
  2356. /* Search for the left edge of the window for each bit */
  2357. for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++) {
  2358. scc_mgr_apply_group_dq_out1_delay(write_group, d);
  2359. writel(0, &sdr_scc_mgr->update);
  2360. /*
  2361. * Stop searching when the read test doesn't pass AND when
  2362. * we've seen a passing read on every bit.
  2363. */
  2364. stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
  2365. 0, PASS_ONE_BIT, &bit_chk, 0);
  2366. sticky_bit_chk = sticky_bit_chk | bit_chk;
  2367. stop = stop && (sticky_bit_chk == param->write_correct_mask);
  2368. debug_cond(DLEVEL == 2, "write_center(left): dtap=%d => %u \
  2369. == %u && %u [bit_chk= %u ]\n",
  2370. d, sticky_bit_chk, param->write_correct_mask,
  2371. stop, bit_chk);
  2372. if (stop == 1) {
  2373. break;
  2374. } else {
  2375. for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
  2376. if (bit_chk & 1) {
  2377. /*
  2378. * Remember a passing test as the
  2379. * left_edge.
  2380. */
  2381. left_edge[i] = d;
  2382. } else {
  2383. /*
  2384. * If a left edge has not been seen
  2385. * yet, then a future passing test will
  2386. * mark this edge as the right edge.
  2387. */
  2388. if (left_edge[i] ==
  2389. IO_IO_OUT1_DELAY_MAX + 1) {
  2390. right_edge[i] = -(d + 1);
  2391. }
  2392. }
  2393. debug_cond(DLEVEL == 2, "write_center[l,d=%d):", d);
  2394. debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
  2395. (int)(bit_chk & 1), i, left_edge[i]);
  2396. debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
  2397. right_edge[i]);
  2398. bit_chk = bit_chk >> 1;
  2399. }
  2400. }
  2401. }
  2402. /* Reset DQ delay chains to 0 */
  2403. scc_mgr_apply_group_dq_out1_delay(0);
  2404. sticky_bit_chk = 0;
  2405. for (i = RW_MGR_MEM_DQ_PER_WRITE_DQS - 1;; i--) {
  2406. debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
  2407. %d right_edge[%u]: %d\n", __func__, __LINE__,
  2408. i, left_edge[i], i, right_edge[i]);
  2409. /*
  2410. * Check for cases where we haven't found the left edge,
  2411. * which makes our assignment of the the right edge invalid.
  2412. * Reset it to the illegal value.
  2413. */
  2414. if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) &&
  2415. (right_edge[i] != IO_IO_OUT1_DELAY_MAX + 1)) {
  2416. right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
  2417. debug_cond(DLEVEL == 2, "%s:%d write_center: reset \
  2418. right_edge[%u]: %d\n", __func__, __LINE__,
  2419. i, right_edge[i]);
  2420. }
  2421. /*
  2422. * Reset sticky bit (except for bits where we have
  2423. * seen the left edge).
  2424. */
  2425. sticky_bit_chk = sticky_bit_chk << 1;
  2426. if ((left_edge[i] != IO_IO_OUT1_DELAY_MAX + 1))
  2427. sticky_bit_chk = sticky_bit_chk | 1;
  2428. if (i == 0)
  2429. break;
  2430. }
  2431. /* Search for the right edge of the window for each bit */
  2432. for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - start_dqs; d++) {
  2433. scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
  2434. d + start_dqs);
  2435. writel(0, &sdr_scc_mgr->update);
  2436. /*
  2437. * Stop searching when the read test doesn't pass AND when
  2438. * we've seen a passing read on every bit.
  2439. */
  2440. stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
  2441. 0, PASS_ONE_BIT, &bit_chk, 0);
  2442. sticky_bit_chk = sticky_bit_chk | bit_chk;
  2443. stop = stop && (sticky_bit_chk == param->write_correct_mask);
  2444. debug_cond(DLEVEL == 2, "write_center (right): dtap=%u => %u == \
  2445. %u && %u\n", d, sticky_bit_chk,
  2446. param->write_correct_mask, stop);
  2447. if (stop == 1) {
  2448. if (d == 0) {
  2449. for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS;
  2450. i++) {
  2451. /* d = 0 failed, but it passed when
  2452. testing the left edge, so it must be
  2453. marginal, set it to -1 */
  2454. if (right_edge[i] ==
  2455. IO_IO_OUT1_DELAY_MAX + 1 &&
  2456. left_edge[i] !=
  2457. IO_IO_OUT1_DELAY_MAX + 1) {
  2458. right_edge[i] = -1;
  2459. }
  2460. }
  2461. }
  2462. break;
  2463. } else {
  2464. for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
  2465. if (bit_chk & 1) {
  2466. /*
  2467. * Remember a passing test as
  2468. * the right_edge.
  2469. */
  2470. right_edge[i] = d;
  2471. } else {
  2472. if (d != 0) {
  2473. /*
  2474. * If a right edge has not
  2475. * been seen yet, then a future
  2476. * passing test will mark this
  2477. * edge as the left edge.
  2478. */
  2479. if (right_edge[i] ==
  2480. IO_IO_OUT1_DELAY_MAX + 1)
  2481. left_edge[i] = -(d + 1);
  2482. } else {
  2483. /*
  2484. * d = 0 failed, but it passed
  2485. * when testing the left edge,
  2486. * so it must be marginal, set
  2487. * it to -1.
  2488. */
  2489. if (right_edge[i] ==
  2490. IO_IO_OUT1_DELAY_MAX + 1 &&
  2491. left_edge[i] !=
  2492. IO_IO_OUT1_DELAY_MAX + 1)
  2493. right_edge[i] = -1;
  2494. /*
  2495. * If a right edge has not been
  2496. * seen yet, then a future
  2497. * passing test will mark this
  2498. * edge as the left edge.
  2499. */
  2500. else if (right_edge[i] ==
  2501. IO_IO_OUT1_DELAY_MAX +
  2502. 1)
  2503. left_edge[i] = -(d + 1);
  2504. }
  2505. }
  2506. debug_cond(DLEVEL == 2, "write_center[r,d=%d):", d);
  2507. debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
  2508. (int)(bit_chk & 1), i, left_edge[i]);
  2509. debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
  2510. right_edge[i]);
  2511. bit_chk = bit_chk >> 1;
  2512. }
  2513. }
  2514. }
  2515. /* Check that all bits have a window */
  2516. for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
  2517. debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
  2518. %d right_edge[%u]: %d", __func__, __LINE__,
  2519. i, left_edge[i], i, right_edge[i]);
  2520. if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) ||
  2521. (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1)) {
  2522. set_failing_group_stage(test_bgn + i,
  2523. CAL_STAGE_WRITES,
  2524. CAL_SUBSTAGE_WRITES_CENTER);
  2525. return 0;
  2526. }
  2527. }
  2528. /* Find middle of window for each DQ bit */
  2529. mid_min = left_edge[0] - right_edge[0];
  2530. min_index = 0;
  2531. for (i = 1; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
  2532. mid = left_edge[i] - right_edge[i];
  2533. if (mid < mid_min) {
  2534. mid_min = mid;
  2535. min_index = i;
  2536. }
  2537. }
  2538. /*
  2539. * -mid_min/2 represents the amount that we need to move DQS.
  2540. * If mid_min is odd and positive we'll need to add one to
  2541. * make sure the rounding in further calculations is correct
  2542. * (always bias to the right), so just add 1 for all positive values.
  2543. */
  2544. if (mid_min > 0)
  2545. mid_min++;
  2546. mid_min = mid_min / 2;
  2547. debug_cond(DLEVEL == 1, "%s:%d write_center: mid_min=%d\n", __func__,
  2548. __LINE__, mid_min);
  2549. /* Determine the amount we can change DQS (which is -mid_min) */
  2550. orig_mid_min = mid_min;
  2551. new_dqs = start_dqs;
  2552. mid_min = 0;
  2553. debug_cond(DLEVEL == 1, "%s:%d write_center: start_dqs=%d new_dqs=%d \
  2554. mid_min=%d\n", __func__, __LINE__, start_dqs, new_dqs, mid_min);
  2555. /* Initialize data for export structures */
  2556. dqs_margin = IO_IO_OUT1_DELAY_MAX + 1;
  2557. dq_margin = IO_IO_OUT1_DELAY_MAX + 1;
  2558. /* add delay to bring centre of all DQ windows to the same "level" */
  2559. for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
  2560. /* Use values before divide by 2 to reduce round off error */
  2561. shift_dq = (left_edge[i] - right_edge[i] -
  2562. (left_edge[min_index] - right_edge[min_index]))/2 +
  2563. (orig_mid_min - mid_min);
  2564. debug_cond(DLEVEL == 2, "%s:%d write_center: before: shift_dq \
  2565. [%u]=%d\n", __func__, __LINE__, i, shift_dq);
  2566. addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
  2567. temp_dq_out1_delay = readl(addr + (i << 2));
  2568. if (shift_dq + (int32_t)temp_dq_out1_delay >
  2569. (int32_t)IO_IO_OUT1_DELAY_MAX) {
  2570. shift_dq = (int32_t)IO_IO_OUT1_DELAY_MAX - temp_dq_out1_delay;
  2571. } else if (shift_dq + (int32_t)temp_dq_out1_delay < 0) {
  2572. shift_dq = -(int32_t)temp_dq_out1_delay;
  2573. }
  2574. debug_cond(DLEVEL == 2, "write_center: after: shift_dq[%u]=%d\n",
  2575. i, shift_dq);
  2576. scc_mgr_set_dq_out1_delay(i, temp_dq_out1_delay + shift_dq);
  2577. scc_mgr_load_dq(i);
  2578. debug_cond(DLEVEL == 2, "write_center: margin[%u]=[%d,%d]\n", i,
  2579. left_edge[i] - shift_dq + (-mid_min),
  2580. right_edge[i] + shift_dq - (-mid_min));
  2581. /* To determine values for export structures */
  2582. if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
  2583. dq_margin = left_edge[i] - shift_dq + (-mid_min);
  2584. if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
  2585. dqs_margin = right_edge[i] + shift_dq - (-mid_min);
  2586. }
  2587. /* Move DQS */
  2588. scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
  2589. writel(0, &sdr_scc_mgr->update);
  2590. /* Centre DM */
  2591. debug_cond(DLEVEL == 2, "%s:%d write_center: DM\n", __func__, __LINE__);
  2592. /*
  2593. * set the left and right edge of each bit to an illegal value,
  2594. * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value,
  2595. */
  2596. left_edge[0] = IO_IO_OUT1_DELAY_MAX + 1;
  2597. right_edge[0] = IO_IO_OUT1_DELAY_MAX + 1;
  2598. int32_t bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
  2599. int32_t end_curr = IO_IO_OUT1_DELAY_MAX + 1;
  2600. int32_t bgn_best = IO_IO_OUT1_DELAY_MAX + 1;
  2601. int32_t end_best = IO_IO_OUT1_DELAY_MAX + 1;
  2602. int32_t win_best = 0;
  2603. /* Search for the/part of the window with DM shift */
  2604. for (d = IO_IO_OUT1_DELAY_MAX; d >= 0; d -= DELTA_D) {
  2605. scc_mgr_apply_group_dm_out1_delay(d);
  2606. writel(0, &sdr_scc_mgr->update);
  2607. if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
  2608. PASS_ALL_BITS, &bit_chk,
  2609. 0)) {
  2610. /* USE Set current end of the window */
  2611. end_curr = -d;
  2612. /*
  2613. * If a starting edge of our window has not been seen
  2614. * this is our current start of the DM window.
  2615. */
  2616. if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
  2617. bgn_curr = -d;
  2618. /*
  2619. * If current window is bigger than best seen.
  2620. * Set best seen to be current window.
  2621. */
  2622. if ((end_curr-bgn_curr+1) > win_best) {
  2623. win_best = end_curr-bgn_curr+1;
  2624. bgn_best = bgn_curr;
  2625. end_best = end_curr;
  2626. }
  2627. } else {
  2628. /* We just saw a failing test. Reset temp edge */
  2629. bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
  2630. end_curr = IO_IO_OUT1_DELAY_MAX + 1;
  2631. }
  2632. }
  2633. /* Reset DM delay chains to 0 */
  2634. scc_mgr_apply_group_dm_out1_delay(0);
  2635. /*
  2636. * Check to see if the current window nudges up aganist 0 delay.
  2637. * If so we need to continue the search by shifting DQS otherwise DQS
  2638. * search begins as a new search. */
  2639. if (end_curr != 0) {
  2640. bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
  2641. end_curr = IO_IO_OUT1_DELAY_MAX + 1;
  2642. }
  2643. /* Search for the/part of the window with DQS shifts */
  2644. for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - new_dqs; d += DELTA_D) {
  2645. /*
  2646. * Note: This only shifts DQS, so are we limiting ourselve to
  2647. * width of DQ unnecessarily.
  2648. */
  2649. scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
  2650. d + new_dqs);
  2651. writel(0, &sdr_scc_mgr->update);
  2652. if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
  2653. PASS_ALL_BITS, &bit_chk,
  2654. 0)) {
  2655. /* USE Set current end of the window */
  2656. end_curr = d;
  2657. /*
  2658. * If a beginning edge of our window has not been seen
  2659. * this is our current begin of the DM window.
  2660. */
  2661. if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
  2662. bgn_curr = d;
  2663. /*
  2664. * If current window is bigger than best seen. Set best
  2665. * seen to be current window.
  2666. */
  2667. if ((end_curr-bgn_curr+1) > win_best) {
  2668. win_best = end_curr-bgn_curr+1;
  2669. bgn_best = bgn_curr;
  2670. end_best = end_curr;
  2671. }
  2672. } else {
  2673. /* We just saw a failing test. Reset temp edge */
  2674. bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
  2675. end_curr = IO_IO_OUT1_DELAY_MAX + 1;
  2676. /* Early exit optimization: if ther remaining delay
  2677. chain space is less than already seen largest window
  2678. we can exit */
  2679. if ((win_best-1) >
  2680. (IO_IO_OUT1_DELAY_MAX - new_dqs - d)) {
  2681. break;
  2682. }
  2683. }
  2684. }
  2685. /* assign left and right edge for cal and reporting; */
  2686. left_edge[0] = -1*bgn_best;
  2687. right_edge[0] = end_best;
  2688. debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d\n", __func__,
  2689. __LINE__, left_edge[0], right_edge[0]);
  2690. /* Move DQS (back to orig) */
  2691. scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
  2692. /* Move DM */
  2693. /* Find middle of window for the DM bit */
  2694. mid = (left_edge[0] - right_edge[0]) / 2;
  2695. /* only move right, since we are not moving DQS/DQ */
  2696. if (mid < 0)
  2697. mid = 0;
  2698. /* dm_marign should fail if we never find a window */
  2699. if (win_best == 0)
  2700. dm_margin = -1;
  2701. else
  2702. dm_margin = left_edge[0] - mid;
  2703. scc_mgr_apply_group_dm_out1_delay(mid);
  2704. writel(0, &sdr_scc_mgr->update);
  2705. debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d mid=%d \
  2706. dm_margin=%d\n", __func__, __LINE__, left_edge[0],
  2707. right_edge[0], mid, dm_margin);
  2708. /* Export values */
  2709. gbl->fom_out += dq_margin + dqs_margin;
  2710. debug_cond(DLEVEL == 2, "%s:%d write_center: dq_margin=%d \
  2711. dqs_margin=%d dm_margin=%d\n", __func__, __LINE__,
  2712. dq_margin, dqs_margin, dm_margin);
  2713. /*
  2714. * Do not remove this line as it makes sure all of our
  2715. * decisions have been applied.
  2716. */
  2717. writel(0, &sdr_scc_mgr->update);
  2718. return (dq_margin >= 0) && (dqs_margin >= 0) && (dm_margin >= 0);
  2719. }
  2720. /* calibrate the write operations */
  2721. static uint32_t rw_mgr_mem_calibrate_writes(uint32_t rank_bgn, uint32_t g,
  2722. uint32_t test_bgn)
  2723. {
  2724. /* update info for sims */
  2725. debug("%s:%d %u %u\n", __func__, __LINE__, g, test_bgn);
  2726. reg_file_set_stage(CAL_STAGE_WRITES);
  2727. reg_file_set_sub_stage(CAL_SUBSTAGE_WRITES_CENTER);
  2728. reg_file_set_group(g);
  2729. if (!rw_mgr_mem_calibrate_writes_center(rank_bgn, g, test_bgn)) {
  2730. set_failing_group_stage(g, CAL_STAGE_WRITES,
  2731. CAL_SUBSTAGE_WRITES_CENTER);
  2732. return 0;
  2733. }
  2734. return 1;
  2735. }
  2736. /**
  2737. * mem_precharge_and_activate() - Precharge all banks and activate
  2738. *
  2739. * Precharge all banks and activate row 0 in bank "000..." and bank "111...".
  2740. */
  2741. static void mem_precharge_and_activate(void)
  2742. {
  2743. int r;
  2744. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
  2745. /* Test if the rank should be skipped. */
  2746. if (param->skip_ranks[r])
  2747. continue;
  2748. /* Set rank. */
  2749. set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
  2750. /* Precharge all banks. */
  2751. writel(RW_MGR_PRECHARGE_ALL, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  2752. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  2753. writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr0);
  2754. writel(RW_MGR_ACTIVATE_0_AND_1_WAIT1,
  2755. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  2756. writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr1);
  2757. writel(RW_MGR_ACTIVATE_0_AND_1_WAIT2,
  2758. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  2759. /* Activate rows. */
  2760. writel(RW_MGR_ACTIVATE_0_AND_1, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  2761. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  2762. }
  2763. }
  2764. /**
  2765. * mem_init_latency() - Configure memory RLAT and WLAT settings
  2766. *
  2767. * Configure memory RLAT and WLAT parameters.
  2768. */
  2769. static void mem_init_latency(void)
  2770. {
  2771. /*
  2772. * For AV/CV, LFIFO is hardened and always runs at full rate
  2773. * so max latency in AFI clocks, used here, is correspondingly
  2774. * smaller.
  2775. */
  2776. const u32 max_latency = (1 << MAX_LATENCY_COUNT_WIDTH) - 1;
  2777. u32 rlat, wlat;
  2778. debug("%s:%d\n", __func__, __LINE__);
  2779. /*
  2780. * Read in write latency.
  2781. * WL for Hard PHY does not include additive latency.
  2782. */
  2783. wlat = readl(&data_mgr->t_wl_add);
  2784. wlat += readl(&data_mgr->mem_t_add);
  2785. gbl->rw_wl_nop_cycles = wlat - 1;
  2786. /* Read in readl latency. */
  2787. rlat = readl(&data_mgr->t_rl_add);
  2788. /* Set a pretty high read latency initially. */
  2789. gbl->curr_read_lat = rlat + 16;
  2790. if (gbl->curr_read_lat > max_latency)
  2791. gbl->curr_read_lat = max_latency;
  2792. writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
  2793. /* Advertise write latency. */
  2794. writel(wlat, &phy_mgr_cfg->afi_wlat);
  2795. }
  2796. /**
  2797. * @mem_skip_calibrate() - Set VFIFO and LFIFO to instant-on settings
  2798. *
  2799. * Set VFIFO and LFIFO to instant-on settings in skip calibration mode.
  2800. */
  2801. static void mem_skip_calibrate(void)
  2802. {
  2803. uint32_t vfifo_offset;
  2804. uint32_t i, j, r;
  2805. debug("%s:%d\n", __func__, __LINE__);
  2806. /* Need to update every shadow register set used by the interface */
  2807. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
  2808. r += NUM_RANKS_PER_SHADOW_REG) {
  2809. /*
  2810. * Set output phase alignment settings appropriate for
  2811. * skip calibration.
  2812. */
  2813. for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
  2814. scc_mgr_set_dqs_en_phase(i, 0);
  2815. #if IO_DLL_CHAIN_LENGTH == 6
  2816. scc_mgr_set_dqdqs_output_phase(i, 6);
  2817. #else
  2818. scc_mgr_set_dqdqs_output_phase(i, 7);
  2819. #endif
  2820. /*
  2821. * Case:33398
  2822. *
  2823. * Write data arrives to the I/O two cycles before write
  2824. * latency is reached (720 deg).
  2825. * -> due to bit-slip in a/c bus
  2826. * -> to allow board skew where dqs is longer than ck
  2827. * -> how often can this happen!?
  2828. * -> can claim back some ptaps for high freq
  2829. * support if we can relax this, but i digress...
  2830. *
  2831. * The write_clk leads mem_ck by 90 deg
  2832. * The minimum ptap of the OPA is 180 deg
  2833. * Each ptap has (360 / IO_DLL_CHAIN_LENGH) deg of delay
  2834. * The write_clk is always delayed by 2 ptaps
  2835. *
  2836. * Hence, to make DQS aligned to CK, we need to delay
  2837. * DQS by:
  2838. * (720 - 90 - 180 - 2 * (360 / IO_DLL_CHAIN_LENGTH))
  2839. *
  2840. * Dividing the above by (360 / IO_DLL_CHAIN_LENGTH)
  2841. * gives us the number of ptaps, which simplies to:
  2842. *
  2843. * (1.25 * IO_DLL_CHAIN_LENGTH - 2)
  2844. */
  2845. scc_mgr_set_dqdqs_output_phase(i,
  2846. 1.25 * IO_DLL_CHAIN_LENGTH - 2);
  2847. }
  2848. writel(0xff, &sdr_scc_mgr->dqs_ena);
  2849. writel(0xff, &sdr_scc_mgr->dqs_io_ena);
  2850. for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
  2851. writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
  2852. SCC_MGR_GROUP_COUNTER_OFFSET);
  2853. }
  2854. writel(0xff, &sdr_scc_mgr->dq_ena);
  2855. writel(0xff, &sdr_scc_mgr->dm_ena);
  2856. writel(0, &sdr_scc_mgr->update);
  2857. }
  2858. /* Compensate for simulation model behaviour */
  2859. for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
  2860. scc_mgr_set_dqs_bus_in_delay(i, 10);
  2861. scc_mgr_load_dqs(i);
  2862. }
  2863. writel(0, &sdr_scc_mgr->update);
  2864. /*
  2865. * ArriaV has hard FIFOs that can only be initialized by incrementing
  2866. * in sequencer.
  2867. */
  2868. vfifo_offset = CALIB_VFIFO_OFFSET;
  2869. for (j = 0; j < vfifo_offset; j++)
  2870. writel(0xff, &phy_mgr_cmd->inc_vfifo_hard_phy);
  2871. writel(0, &phy_mgr_cmd->fifo_reset);
  2872. /*
  2873. * For Arria V and Cyclone V with hard LFIFO, we get the skip-cal
  2874. * setting from generation-time constant.
  2875. */
  2876. gbl->curr_read_lat = CALIB_LFIFO_OFFSET;
  2877. writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
  2878. }
  2879. /**
  2880. * mem_calibrate() - Memory calibration entry point.
  2881. *
  2882. * Perform memory calibration.
  2883. */
  2884. static uint32_t mem_calibrate(void)
  2885. {
  2886. uint32_t i;
  2887. uint32_t rank_bgn, sr;
  2888. uint32_t write_group, write_test_bgn;
  2889. uint32_t read_group, read_test_bgn;
  2890. uint32_t run_groups, current_run;
  2891. uint32_t failing_groups = 0;
  2892. uint32_t group_failed = 0;
  2893. const u32 rwdqs_ratio = RW_MGR_MEM_IF_READ_DQS_WIDTH /
  2894. RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
  2895. debug("%s:%d\n", __func__, __LINE__);
  2896. /* Initialize the data settings */
  2897. gbl->error_substage = CAL_SUBSTAGE_NIL;
  2898. gbl->error_stage = CAL_STAGE_NIL;
  2899. gbl->error_group = 0xff;
  2900. gbl->fom_in = 0;
  2901. gbl->fom_out = 0;
  2902. /* Initialize WLAT and RLAT. */
  2903. mem_init_latency();
  2904. /* Initialize bit slips. */
  2905. mem_precharge_and_activate();
  2906. for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
  2907. writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
  2908. SCC_MGR_GROUP_COUNTER_OFFSET);
  2909. /* Only needed once to set all groups, pins, DQ, DQS, DM. */
  2910. if (i == 0)
  2911. scc_mgr_set_hhp_extras();
  2912. scc_set_bypass_mode(i);
  2913. }
  2914. /* Calibration is skipped. */
  2915. if ((dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) {
  2916. /*
  2917. * Set VFIFO and LFIFO to instant-on settings in skip
  2918. * calibration mode.
  2919. */
  2920. mem_skip_calibrate();
  2921. /*
  2922. * Do not remove this line as it makes sure all of our
  2923. * decisions have been applied.
  2924. */
  2925. writel(0, &sdr_scc_mgr->update);
  2926. return 1;
  2927. }
  2928. /* Calibration is not skipped. */
  2929. for (i = 0; i < NUM_CALIB_REPEAT; i++) {
  2930. /*
  2931. * Zero all delay chain/phase settings for all
  2932. * groups and all shadow register sets.
  2933. */
  2934. scc_mgr_zero_all();
  2935. run_groups = ~param->skip_groups;
  2936. for (write_group = 0, write_test_bgn = 0; write_group
  2937. < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; write_group++,
  2938. write_test_bgn += RW_MGR_MEM_DQ_PER_WRITE_DQS) {
  2939. /* Initialize the group failure */
  2940. group_failed = 0;
  2941. current_run = run_groups & ((1 <<
  2942. RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1);
  2943. run_groups = run_groups >>
  2944. RW_MGR_NUM_DQS_PER_WRITE_GROUP;
  2945. if (current_run == 0)
  2946. continue;
  2947. writel(write_group, SDR_PHYGRP_SCCGRP_ADDRESS |
  2948. SCC_MGR_GROUP_COUNTER_OFFSET);
  2949. scc_mgr_zero_group(write_group, 0);
  2950. for (read_group = write_group * rwdqs_ratio,
  2951. read_test_bgn = 0;
  2952. read_group < (write_group + 1) * rwdqs_ratio;
  2953. read_group++,
  2954. read_test_bgn += RW_MGR_MEM_DQ_PER_READ_DQS) {
  2955. if (STATIC_CALIB_STEPS & CALIB_SKIP_VFIFO)
  2956. continue;
  2957. /* Calibrate the VFIFO */
  2958. if (rw_mgr_mem_calibrate_vfifo(read_group,
  2959. read_test_bgn))
  2960. continue;
  2961. if (!(gbl->phy_debug_mode_flags & PHY_DEBUG_SWEEP_ALL_GROUPS))
  2962. return 0;
  2963. /* The group failed, we're done. */
  2964. goto grp_failed;
  2965. }
  2966. /* Calibrate the output side */
  2967. for (rank_bgn = 0, sr = 0;
  2968. rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
  2969. rank_bgn += NUM_RANKS_PER_SHADOW_REG, sr++) {
  2970. if (STATIC_CALIB_STEPS & CALIB_SKIP_WRITES)
  2971. continue;
  2972. /* Not needed in quick mode! */
  2973. if (STATIC_CALIB_STEPS & CALIB_SKIP_DELAY_SWEEPS)
  2974. continue;
  2975. /*
  2976. * Determine if this set of ranks
  2977. * should be skipped entirely.
  2978. */
  2979. if (param->skip_shadow_regs[sr])
  2980. continue;
  2981. /* Calibrate WRITEs */
  2982. if (rw_mgr_mem_calibrate_writes(rank_bgn,
  2983. write_group, write_test_bgn))
  2984. continue;
  2985. group_failed = 1;
  2986. if (!(gbl->phy_debug_mode_flags & PHY_DEBUG_SWEEP_ALL_GROUPS))
  2987. return 0;
  2988. }
  2989. /* Some group failed, we're done. */
  2990. if (group_failed)
  2991. goto grp_failed;
  2992. for (read_group = write_group * rwdqs_ratio,
  2993. read_test_bgn = 0;
  2994. read_group < (write_group + 1) * rwdqs_ratio;
  2995. read_group++,
  2996. read_test_bgn += RW_MGR_MEM_DQ_PER_READ_DQS) {
  2997. if (STATIC_CALIB_STEPS & CALIB_SKIP_WRITES)
  2998. continue;
  2999. if (rw_mgr_mem_calibrate_vfifo_end(read_group,
  3000. read_test_bgn))
  3001. continue;
  3002. if (!(gbl->phy_debug_mode_flags & PHY_DEBUG_SWEEP_ALL_GROUPS))
  3003. return 0;
  3004. /* The group failed, we're done. */
  3005. goto grp_failed;
  3006. }
  3007. /* No group failed, continue as usual. */
  3008. continue;
  3009. grp_failed: /* A group failed, increment the counter. */
  3010. failing_groups++;
  3011. }
  3012. /*
  3013. * USER If there are any failing groups then report
  3014. * the failure.
  3015. */
  3016. if (failing_groups != 0)
  3017. return 0;
  3018. if (STATIC_CALIB_STEPS & CALIB_SKIP_LFIFO)
  3019. continue;
  3020. /*
  3021. * If we're skipping groups as part of debug,
  3022. * don't calibrate LFIFO.
  3023. */
  3024. if (param->skip_groups != 0)
  3025. continue;
  3026. /* Calibrate the LFIFO */
  3027. if (!rw_mgr_mem_calibrate_lfifo())
  3028. return 0;
  3029. }
  3030. /*
  3031. * Do not remove this line as it makes sure all of our decisions
  3032. * have been applied.
  3033. */
  3034. writel(0, &sdr_scc_mgr->update);
  3035. return 1;
  3036. }
  3037. /**
  3038. * run_mem_calibrate() - Perform memory calibration
  3039. *
  3040. * This function triggers the entire memory calibration procedure.
  3041. */
  3042. static int run_mem_calibrate(void)
  3043. {
  3044. int pass;
  3045. debug("%s:%d\n", __func__, __LINE__);
  3046. /* Reset pass/fail status shown on afi_cal_success/fail */
  3047. writel(PHY_MGR_CAL_RESET, &phy_mgr_cfg->cal_status);
  3048. /* Stop tracking manager. */
  3049. clrbits_le32(&sdr_ctrl->ctrl_cfg, 1 << 22);
  3050. phy_mgr_initialize();
  3051. rw_mgr_mem_initialize();
  3052. /* Perform the actual memory calibration. */
  3053. pass = mem_calibrate();
  3054. mem_precharge_and_activate();
  3055. writel(0, &phy_mgr_cmd->fifo_reset);
  3056. /* Handoff. */
  3057. rw_mgr_mem_handoff();
  3058. /*
  3059. * In Hard PHY this is a 2-bit control:
  3060. * 0: AFI Mux Select
  3061. * 1: DDIO Mux Select
  3062. */
  3063. writel(0x2, &phy_mgr_cfg->mux_sel);
  3064. /* Start tracking manager. */
  3065. setbits_le32(&sdr_ctrl->ctrl_cfg, 1 << 22);
  3066. return pass;
  3067. }
  3068. /**
  3069. * debug_mem_calibrate() - Report result of memory calibration
  3070. * @pass: Value indicating whether calibration passed or failed
  3071. *
  3072. * This function reports the results of the memory calibration
  3073. * and writes debug information into the register file.
  3074. */
  3075. static void debug_mem_calibrate(int pass)
  3076. {
  3077. uint32_t debug_info;
  3078. if (pass) {
  3079. printf("%s: CALIBRATION PASSED\n", __FILE__);
  3080. gbl->fom_in /= 2;
  3081. gbl->fom_out /= 2;
  3082. if (gbl->fom_in > 0xff)
  3083. gbl->fom_in = 0xff;
  3084. if (gbl->fom_out > 0xff)
  3085. gbl->fom_out = 0xff;
  3086. /* Update the FOM in the register file */
  3087. debug_info = gbl->fom_in;
  3088. debug_info |= gbl->fom_out << 8;
  3089. writel(debug_info, &sdr_reg_file->fom);
  3090. writel(debug_info, &phy_mgr_cfg->cal_debug_info);
  3091. writel(PHY_MGR_CAL_SUCCESS, &phy_mgr_cfg->cal_status);
  3092. } else {
  3093. printf("%s: CALIBRATION FAILED\n", __FILE__);
  3094. debug_info = gbl->error_stage;
  3095. debug_info |= gbl->error_substage << 8;
  3096. debug_info |= gbl->error_group << 16;
  3097. writel(debug_info, &sdr_reg_file->failing_stage);
  3098. writel(debug_info, &phy_mgr_cfg->cal_debug_info);
  3099. writel(PHY_MGR_CAL_FAIL, &phy_mgr_cfg->cal_status);
  3100. /* Update the failing group/stage in the register file */
  3101. debug_info = gbl->error_stage;
  3102. debug_info |= gbl->error_substage << 8;
  3103. debug_info |= gbl->error_group << 16;
  3104. writel(debug_info, &sdr_reg_file->failing_stage);
  3105. }
  3106. printf("%s: Calibration complete\n", __FILE__);
  3107. }
  3108. /**
  3109. * hc_initialize_rom_data() - Initialize ROM data
  3110. *
  3111. * Initialize ROM data.
  3112. */
  3113. static void hc_initialize_rom_data(void)
  3114. {
  3115. u32 i, addr;
  3116. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_INST_ROM_WRITE_OFFSET;
  3117. for (i = 0; i < ARRAY_SIZE(inst_rom_init); i++)
  3118. writel(inst_rom_init[i], addr + (i << 2));
  3119. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_AC_ROM_WRITE_OFFSET;
  3120. for (i = 0; i < ARRAY_SIZE(ac_rom_init); i++)
  3121. writel(ac_rom_init[i], addr + (i << 2));
  3122. }
  3123. /**
  3124. * initialize_reg_file() - Initialize SDR register file
  3125. *
  3126. * Initialize SDR register file.
  3127. */
  3128. static void initialize_reg_file(void)
  3129. {
  3130. /* Initialize the register file with the correct data */
  3131. writel(REG_FILE_INIT_SEQ_SIGNATURE, &sdr_reg_file->signature);
  3132. writel(0, &sdr_reg_file->debug_data_addr);
  3133. writel(0, &sdr_reg_file->cur_stage);
  3134. writel(0, &sdr_reg_file->fom);
  3135. writel(0, &sdr_reg_file->failing_stage);
  3136. writel(0, &sdr_reg_file->debug1);
  3137. writel(0, &sdr_reg_file->debug2);
  3138. }
  3139. /**
  3140. * initialize_hps_phy() - Initialize HPS PHY
  3141. *
  3142. * Initialize HPS PHY.
  3143. */
  3144. static void initialize_hps_phy(void)
  3145. {
  3146. uint32_t reg;
  3147. /*
  3148. * Tracking also gets configured here because it's in the
  3149. * same register.
  3150. */
  3151. uint32_t trk_sample_count = 7500;
  3152. uint32_t trk_long_idle_sample_count = (10 << 16) | 100;
  3153. /*
  3154. * Format is number of outer loops in the 16 MSB, sample
  3155. * count in 16 LSB.
  3156. */
  3157. reg = 0;
  3158. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(2);
  3159. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQDELAYEN_SET(1);
  3160. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSDELAYEN_SET(1);
  3161. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSLOGICDELAYEN_SET(1);
  3162. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_RESETDELAYEN_SET(0);
  3163. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(1);
  3164. /*
  3165. * This field selects the intrinsic latency to RDATA_EN/FULL path.
  3166. * 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles.
  3167. */
  3168. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ADDLATSEL_SET(0);
  3169. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_SET(
  3170. trk_sample_count);
  3171. writel(reg, &sdr_ctrl->phy_ctrl0);
  3172. reg = 0;
  3173. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_SAMPLECOUNT_31_20_SET(
  3174. trk_sample_count >>
  3175. SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_WIDTH);
  3176. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_SET(
  3177. trk_long_idle_sample_count);
  3178. writel(reg, &sdr_ctrl->phy_ctrl1);
  3179. reg = 0;
  3180. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_LONGIDLESAMPLECOUNT_31_20_SET(
  3181. trk_long_idle_sample_count >>
  3182. SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_WIDTH);
  3183. writel(reg, &sdr_ctrl->phy_ctrl2);
  3184. }
  3185. /**
  3186. * initialize_tracking() - Initialize tracking
  3187. *
  3188. * Initialize the register file with usable initial data.
  3189. */
  3190. static void initialize_tracking(void)
  3191. {
  3192. /*
  3193. * Initialize the register file with the correct data.
  3194. * Compute usable version of value in case we skip full
  3195. * computation later.
  3196. */
  3197. writel(DIV_ROUND_UP(IO_DELAY_PER_OPA_TAP, IO_DELAY_PER_DCHAIN_TAP) - 1,
  3198. &sdr_reg_file->dtaps_per_ptap);
  3199. /* trk_sample_count */
  3200. writel(7500, &sdr_reg_file->trk_sample_count);
  3201. /* longidle outer loop [15:0] */
  3202. writel((10 << 16) | (100 << 0), &sdr_reg_file->trk_longidle);
  3203. /*
  3204. * longidle sample count [31:24]
  3205. * trfc, worst case of 933Mhz 4Gb [23:16]
  3206. * trcd, worst case [15:8]
  3207. * vfifo wait [7:0]
  3208. */
  3209. writel((243 << 24) | (14 << 16) | (10 << 8) | (4 << 0),
  3210. &sdr_reg_file->delays);
  3211. /* mux delay */
  3212. writel((RW_MGR_IDLE << 24) | (RW_MGR_ACTIVATE_1 << 16) |
  3213. (RW_MGR_SGLE_READ << 8) | (RW_MGR_PRECHARGE_ALL << 0),
  3214. &sdr_reg_file->trk_rw_mgr_addr);
  3215. writel(RW_MGR_MEM_IF_READ_DQS_WIDTH,
  3216. &sdr_reg_file->trk_read_dqs_width);
  3217. /* trefi [7:0] */
  3218. writel((RW_MGR_REFRESH_ALL << 24) | (1000 << 0),
  3219. &sdr_reg_file->trk_rfsh);
  3220. }
  3221. int sdram_calibration_full(void)
  3222. {
  3223. struct param_type my_param;
  3224. struct gbl_type my_gbl;
  3225. uint32_t pass;
  3226. memset(&my_param, 0, sizeof(my_param));
  3227. memset(&my_gbl, 0, sizeof(my_gbl));
  3228. param = &my_param;
  3229. gbl = &my_gbl;
  3230. /* Set the calibration enabled by default */
  3231. gbl->phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT;
  3232. /*
  3233. * Only sweep all groups (regardless of fail state) by default
  3234. * Set enabled read test by default.
  3235. */
  3236. #if DISABLE_GUARANTEED_READ
  3237. gbl->phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ;
  3238. #endif
  3239. /* Initialize the register file */
  3240. initialize_reg_file();
  3241. /* Initialize any PHY CSR */
  3242. initialize_hps_phy();
  3243. scc_mgr_initialize();
  3244. initialize_tracking();
  3245. printf("%s: Preparing to start memory calibration\n", __FILE__);
  3246. debug("%s:%d\n", __func__, __LINE__);
  3247. debug_cond(DLEVEL == 1,
  3248. "DDR3 FULL_RATE ranks=%u cs/dimm=%u dq/dqs=%u,%u vg/dqs=%u,%u ",
  3249. RW_MGR_MEM_NUMBER_OF_RANKS, RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM,
  3250. RW_MGR_MEM_DQ_PER_READ_DQS, RW_MGR_MEM_DQ_PER_WRITE_DQS,
  3251. RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS,
  3252. RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
  3253. debug_cond(DLEVEL == 1,
  3254. "dqs=%u,%u dq=%u dm=%u ptap_delay=%u dtap_delay=%u ",
  3255. RW_MGR_MEM_IF_READ_DQS_WIDTH, RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
  3256. RW_MGR_MEM_DATA_WIDTH, RW_MGR_MEM_DATA_MASK_WIDTH,
  3257. IO_DELAY_PER_OPA_TAP, IO_DELAY_PER_DCHAIN_TAP);
  3258. debug_cond(DLEVEL == 1, "dtap_dqsen_delay=%u, dll=%u",
  3259. IO_DELAY_PER_DQS_EN_DCHAIN_TAP, IO_DLL_CHAIN_LENGTH);
  3260. debug_cond(DLEVEL == 1, "max values: en_p=%u dqdqs_p=%u en_d=%u dqs_in_d=%u ",
  3261. IO_DQS_EN_PHASE_MAX, IO_DQDQS_OUT_PHASE_MAX,
  3262. IO_DQS_EN_DELAY_MAX, IO_DQS_IN_DELAY_MAX);
  3263. debug_cond(DLEVEL == 1, "io_in_d=%u io_out1_d=%u io_out2_d=%u ",
  3264. IO_IO_IN_DELAY_MAX, IO_IO_OUT1_DELAY_MAX,
  3265. IO_IO_OUT2_DELAY_MAX);
  3266. debug_cond(DLEVEL == 1, "dqs_in_reserve=%u dqs_out_reserve=%u\n",
  3267. IO_DQS_IN_RESERVE, IO_DQS_OUT_RESERVE);
  3268. hc_initialize_rom_data();
  3269. /* update info for sims */
  3270. reg_file_set_stage(CAL_STAGE_NIL);
  3271. reg_file_set_group(0);
  3272. /*
  3273. * Load global needed for those actions that require
  3274. * some dynamic calibration support.
  3275. */
  3276. dyn_calib_steps = STATIC_CALIB_STEPS;
  3277. /*
  3278. * Load global to allow dynamic selection of delay loop settings
  3279. * based on calibration mode.
  3280. */
  3281. if (!(dyn_calib_steps & CALIB_SKIP_DELAY_LOOPS))
  3282. skip_delay_mask = 0xff;
  3283. else
  3284. skip_delay_mask = 0x0;
  3285. pass = run_mem_calibrate();
  3286. debug_mem_calibrate(pass);
  3287. return pass;
  3288. }