sequencer.c 107 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766
  1. /*
  2. * Copyright Altera Corporation (C) 2012-2015
  3. *
  4. * SPDX-License-Identifier: BSD-3-Clause
  5. */
  6. #include <common.h>
  7. #include <asm/io.h>
  8. #include <asm/arch/sdram.h>
  9. #include "sequencer.h"
  10. #include "sequencer_auto.h"
  11. #include "sequencer_auto_ac_init.h"
  12. #include "sequencer_auto_inst_init.h"
  13. #include "sequencer_defines.h"
  14. static struct socfpga_sdr_rw_load_manager *sdr_rw_load_mgr_regs =
  15. (struct socfpga_sdr_rw_load_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0x800);
  16. static struct socfpga_sdr_rw_load_jump_manager *sdr_rw_load_jump_mgr_regs =
  17. (struct socfpga_sdr_rw_load_jump_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0xC00);
  18. static struct socfpga_sdr_reg_file *sdr_reg_file =
  19. (struct socfpga_sdr_reg_file *)SDR_PHYGRP_REGFILEGRP_ADDRESS;
  20. static struct socfpga_sdr_scc_mgr *sdr_scc_mgr =
  21. (struct socfpga_sdr_scc_mgr *)(SDR_PHYGRP_SCCGRP_ADDRESS | 0xe00);
  22. static struct socfpga_phy_mgr_cmd *phy_mgr_cmd =
  23. (struct socfpga_phy_mgr_cmd *)SDR_PHYGRP_PHYMGRGRP_ADDRESS;
  24. static struct socfpga_phy_mgr_cfg *phy_mgr_cfg =
  25. (struct socfpga_phy_mgr_cfg *)(SDR_PHYGRP_PHYMGRGRP_ADDRESS | 0x40);
  26. static struct socfpga_data_mgr *data_mgr =
  27. (struct socfpga_data_mgr *)SDR_PHYGRP_DATAMGRGRP_ADDRESS;
  28. static struct socfpga_sdr_ctrl *sdr_ctrl =
  29. (struct socfpga_sdr_ctrl *)SDR_CTRLGRP_ADDRESS;
  30. #define DELTA_D 1
  31. /*
  32. * In order to reduce ROM size, most of the selectable calibration steps are
  33. * decided at compile time based on the user's calibration mode selection,
  34. * as captured by the STATIC_CALIB_STEPS selection below.
  35. *
  36. * However, to support simulation-time selection of fast simulation mode, where
  37. * we skip everything except the bare minimum, we need a few of the steps to
  38. * be dynamic. In those cases, we either use the DYNAMIC_CALIB_STEPS for the
  39. * check, which is based on the rtl-supplied value, or we dynamically compute
  40. * the value to use based on the dynamically-chosen calibration mode
  41. */
  42. #define DLEVEL 0
  43. #define STATIC_IN_RTL_SIM 0
  44. #define STATIC_SKIP_DELAY_LOOPS 0
  45. #define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | \
  46. STATIC_SKIP_DELAY_LOOPS)
  47. /* calibration steps requested by the rtl */
  48. uint16_t dyn_calib_steps;
  49. /*
  50. * To make CALIB_SKIP_DELAY_LOOPS a dynamic conditional option
  51. * instead of static, we use boolean logic to select between
  52. * non-skip and skip values
  53. *
  54. * The mask is set to include all bits when not-skipping, but is
  55. * zero when skipping
  56. */
  57. uint16_t skip_delay_mask; /* mask off bits when skipping/not-skipping */
  58. #define SKIP_DELAY_LOOP_VALUE_OR_ZERO(non_skip_value) \
  59. ((non_skip_value) & skip_delay_mask)
  60. struct gbl_type *gbl;
  61. struct param_type *param;
  62. uint32_t curr_shadow_reg;
  63. static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
  64. uint32_t write_group, uint32_t use_dm,
  65. uint32_t all_correct, uint32_t *bit_chk, uint32_t all_ranks);
  66. static void set_failing_group_stage(uint32_t group, uint32_t stage,
  67. uint32_t substage)
  68. {
  69. /*
  70. * Only set the global stage if there was not been any other
  71. * failing group
  72. */
  73. if (gbl->error_stage == CAL_STAGE_NIL) {
  74. gbl->error_substage = substage;
  75. gbl->error_stage = stage;
  76. gbl->error_group = group;
  77. }
  78. }
  79. static void reg_file_set_group(u16 set_group)
  80. {
  81. clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff0000, set_group << 16);
  82. }
  83. static void reg_file_set_stage(u8 set_stage)
  84. {
  85. clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff, set_stage & 0xff);
  86. }
  87. static void reg_file_set_sub_stage(u8 set_sub_stage)
  88. {
  89. set_sub_stage &= 0xff;
  90. clrsetbits_le32(&sdr_reg_file->cur_stage, 0xff00, set_sub_stage << 8);
  91. }
  92. static void initialize(void)
  93. {
  94. debug("%s:%d\n", __func__, __LINE__);
  95. /* USER calibration has control over path to memory */
  96. /*
  97. * In Hard PHY this is a 2-bit control:
  98. * 0: AFI Mux Select
  99. * 1: DDIO Mux Select
  100. */
  101. writel(0x3, &phy_mgr_cfg->mux_sel);
  102. /* USER memory clock is not stable we begin initialization */
  103. writel(0, &phy_mgr_cfg->reset_mem_stbl);
  104. /* USER calibration status all set to zero */
  105. writel(0, &phy_mgr_cfg->cal_status);
  106. writel(0, &phy_mgr_cfg->cal_debug_info);
  107. if ((dyn_calib_steps & CALIB_SKIP_ALL) != CALIB_SKIP_ALL) {
  108. param->read_correct_mask_vg = ((uint32_t)1 <<
  109. (RW_MGR_MEM_DQ_PER_READ_DQS /
  110. RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS)) - 1;
  111. param->write_correct_mask_vg = ((uint32_t)1 <<
  112. (RW_MGR_MEM_DQ_PER_READ_DQS /
  113. RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS)) - 1;
  114. param->read_correct_mask = ((uint32_t)1 <<
  115. RW_MGR_MEM_DQ_PER_READ_DQS) - 1;
  116. param->write_correct_mask = ((uint32_t)1 <<
  117. RW_MGR_MEM_DQ_PER_WRITE_DQS) - 1;
  118. param->dm_correct_mask = ((uint32_t)1 <<
  119. (RW_MGR_MEM_DATA_WIDTH / RW_MGR_MEM_DATA_MASK_WIDTH))
  120. - 1;
  121. }
  122. }
  123. static void set_rank_and_odt_mask(uint32_t rank, uint32_t odt_mode)
  124. {
  125. uint32_t odt_mask_0 = 0;
  126. uint32_t odt_mask_1 = 0;
  127. uint32_t cs_and_odt_mask;
  128. if (odt_mode == RW_MGR_ODT_MODE_READ_WRITE) {
  129. if (RW_MGR_MEM_NUMBER_OF_RANKS == 1) {
  130. /*
  131. * 1 Rank
  132. * Read: ODT = 0
  133. * Write: ODT = 1
  134. */
  135. odt_mask_0 = 0x0;
  136. odt_mask_1 = 0x1;
  137. } else if (RW_MGR_MEM_NUMBER_OF_RANKS == 2) {
  138. /* 2 Ranks */
  139. if (RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 1) {
  140. /* - Dual-Slot , Single-Rank
  141. * (1 chip-select per DIMM)
  142. * OR
  143. * - RDIMM, 4 total CS (2 CS per DIMM)
  144. * means 2 DIMM
  145. * Since MEM_NUMBER_OF_RANKS is 2 they are
  146. * both single rank
  147. * with 2 CS each (special for RDIMM)
  148. * Read: Turn on ODT on the opposite rank
  149. * Write: Turn on ODT on all ranks
  150. */
  151. odt_mask_0 = 0x3 & ~(1 << rank);
  152. odt_mask_1 = 0x3;
  153. } else {
  154. /*
  155. * USER - Single-Slot , Dual-rank DIMMs
  156. * (2 chip-selects per DIMM)
  157. * USER Read: Turn on ODT off on all ranks
  158. * USER Write: Turn on ODT on active rank
  159. */
  160. odt_mask_0 = 0x0;
  161. odt_mask_1 = 0x3 & (1 << rank);
  162. }
  163. } else {
  164. /* 4 Ranks
  165. * Read:
  166. * ----------+-----------------------+
  167. * | |
  168. * | ODT |
  169. * Read From +-----------------------+
  170. * Rank | 3 | 2 | 1 | 0 |
  171. * ----------+-----+-----+-----+-----+
  172. * 0 | 0 | 1 | 0 | 0 |
  173. * 1 | 1 | 0 | 0 | 0 |
  174. * 2 | 0 | 0 | 0 | 1 |
  175. * 3 | 0 | 0 | 1 | 0 |
  176. * ----------+-----+-----+-----+-----+
  177. *
  178. * Write:
  179. * ----------+-----------------------+
  180. * | |
  181. * | ODT |
  182. * Write To +-----------------------+
  183. * Rank | 3 | 2 | 1 | 0 |
  184. * ----------+-----+-----+-----+-----+
  185. * 0 | 0 | 1 | 0 | 1 |
  186. * 1 | 1 | 0 | 1 | 0 |
  187. * 2 | 0 | 1 | 0 | 1 |
  188. * 3 | 1 | 0 | 1 | 0 |
  189. * ----------+-----+-----+-----+-----+
  190. */
  191. switch (rank) {
  192. case 0:
  193. odt_mask_0 = 0x4;
  194. odt_mask_1 = 0x5;
  195. break;
  196. case 1:
  197. odt_mask_0 = 0x8;
  198. odt_mask_1 = 0xA;
  199. break;
  200. case 2:
  201. odt_mask_0 = 0x1;
  202. odt_mask_1 = 0x5;
  203. break;
  204. case 3:
  205. odt_mask_0 = 0x2;
  206. odt_mask_1 = 0xA;
  207. break;
  208. }
  209. }
  210. } else {
  211. odt_mask_0 = 0x0;
  212. odt_mask_1 = 0x0;
  213. }
  214. cs_and_odt_mask =
  215. (0xFF & ~(1 << rank)) |
  216. ((0xFF & odt_mask_0) << 8) |
  217. ((0xFF & odt_mask_1) << 16);
  218. writel(cs_and_odt_mask, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  219. RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
  220. }
  221. /**
  222. * scc_mgr_set() - Set SCC Manager register
  223. * @off: Base offset in SCC Manager space
  224. * @grp: Read/Write group
  225. * @val: Value to be set
  226. *
  227. * This function sets the SCC Manager (Scan Chain Control Manager) register.
  228. */
  229. static void scc_mgr_set(u32 off, u32 grp, u32 val)
  230. {
  231. writel(val, SDR_PHYGRP_SCCGRP_ADDRESS | off | (grp << 2));
  232. }
  233. /**
  234. * scc_mgr_initialize() - Initialize SCC Manager registers
  235. *
  236. * Initialize SCC Manager registers.
  237. */
  238. static void scc_mgr_initialize(void)
  239. {
  240. /*
  241. * Clear register file for HPS. 16 (2^4) is the size of the
  242. * full register file in the scc mgr:
  243. * RFILE_DEPTH = 1 + log2(MEM_DQ_PER_DQS + 1 + MEM_DM_PER_DQS +
  244. * MEM_IF_READ_DQS_WIDTH - 1);
  245. */
  246. int i;
  247. for (i = 0; i < 16; i++) {
  248. debug_cond(DLEVEL == 1, "%s:%d: Clearing SCC RFILE index %u\n",
  249. __func__, __LINE__, i);
  250. scc_mgr_set(SCC_MGR_HHP_RFILE_OFFSET, 0, i);
  251. }
  252. }
  253. static void scc_mgr_set_dqdqs_output_phase(uint32_t write_group, uint32_t phase)
  254. {
  255. scc_mgr_set(SCC_MGR_DQDQS_OUT_PHASE_OFFSET, write_group, phase);
  256. }
  257. static void scc_mgr_set_dqs_bus_in_delay(uint32_t read_group, uint32_t delay)
  258. {
  259. scc_mgr_set(SCC_MGR_DQS_IN_DELAY_OFFSET, read_group, delay);
  260. }
  261. static void scc_mgr_set_dqs_en_phase(uint32_t read_group, uint32_t phase)
  262. {
  263. scc_mgr_set(SCC_MGR_DQS_EN_PHASE_OFFSET, read_group, phase);
  264. }
  265. static void scc_mgr_set_dqs_en_delay(uint32_t read_group, uint32_t delay)
  266. {
  267. scc_mgr_set(SCC_MGR_DQS_EN_DELAY_OFFSET, read_group, delay);
  268. }
  269. static void scc_mgr_set_dqs_io_in_delay(uint32_t delay)
  270. {
  271. scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, RW_MGR_MEM_DQ_PER_WRITE_DQS,
  272. delay);
  273. }
  274. static void scc_mgr_set_dq_in_delay(uint32_t dq_in_group, uint32_t delay)
  275. {
  276. scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, dq_in_group, delay);
  277. }
  278. static void scc_mgr_set_dq_out1_delay(uint32_t dq_in_group, uint32_t delay)
  279. {
  280. scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET, dq_in_group, delay);
  281. }
  282. static void scc_mgr_set_dqs_out1_delay(uint32_t delay)
  283. {
  284. scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET, RW_MGR_MEM_DQ_PER_WRITE_DQS,
  285. delay);
  286. }
  287. static void scc_mgr_set_dm_out1_delay(uint32_t dm, uint32_t delay)
  288. {
  289. scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET,
  290. RW_MGR_MEM_DQ_PER_WRITE_DQS + 1 + dm,
  291. delay);
  292. }
  293. /* load up dqs config settings */
  294. static void scc_mgr_load_dqs(uint32_t dqs)
  295. {
  296. writel(dqs, &sdr_scc_mgr->dqs_ena);
  297. }
  298. /* load up dqs io config settings */
  299. static void scc_mgr_load_dqs_io(void)
  300. {
  301. writel(0, &sdr_scc_mgr->dqs_io_ena);
  302. }
  303. /* load up dq config settings */
  304. static void scc_mgr_load_dq(uint32_t dq_in_group)
  305. {
  306. writel(dq_in_group, &sdr_scc_mgr->dq_ena);
  307. }
  308. /* load up dm config settings */
  309. static void scc_mgr_load_dm(uint32_t dm)
  310. {
  311. writel(dm, &sdr_scc_mgr->dm_ena);
  312. }
  313. /**
  314. * scc_mgr_set_all_ranks() - Set SCC Manager register for all ranks
  315. * @off: Base offset in SCC Manager space
  316. * @grp: Read/Write group
  317. * @val: Value to be set
  318. * @update: If non-zero, trigger SCC Manager update for all ranks
  319. *
  320. * This function sets the SCC Manager (Scan Chain Control Manager) register
  321. * and optionally triggers the SCC update for all ranks.
  322. */
  323. static void scc_mgr_set_all_ranks(const u32 off, const u32 grp, const u32 val,
  324. const int update)
  325. {
  326. u32 r;
  327. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
  328. r += NUM_RANKS_PER_SHADOW_REG) {
  329. scc_mgr_set(off, grp, val);
  330. if (update || (r == 0)) {
  331. writel(grp, &sdr_scc_mgr->dqs_ena);
  332. writel(0, &sdr_scc_mgr->update);
  333. }
  334. }
  335. }
  336. static void scc_mgr_set_dqs_en_phase_all_ranks(u32 read_group, u32 phase)
  337. {
  338. /*
  339. * USER although the h/w doesn't support different phases per
  340. * shadow register, for simplicity our scc manager modeling
  341. * keeps different phase settings per shadow reg, and it's
  342. * important for us to keep them in sync to match h/w.
  343. * for efficiency, the scan chain update should occur only
  344. * once to sr0.
  345. */
  346. scc_mgr_set_all_ranks(SCC_MGR_DQS_EN_PHASE_OFFSET,
  347. read_group, phase, 0);
  348. }
  349. static void scc_mgr_set_dqdqs_output_phase_all_ranks(uint32_t write_group,
  350. uint32_t phase)
  351. {
  352. /*
  353. * USER although the h/w doesn't support different phases per
  354. * shadow register, for simplicity our scc manager modeling
  355. * keeps different phase settings per shadow reg, and it's
  356. * important for us to keep them in sync to match h/w.
  357. * for efficiency, the scan chain update should occur only
  358. * once to sr0.
  359. */
  360. scc_mgr_set_all_ranks(SCC_MGR_DQDQS_OUT_PHASE_OFFSET,
  361. write_group, phase, 0);
  362. }
  363. static void scc_mgr_set_dqs_en_delay_all_ranks(uint32_t read_group,
  364. uint32_t delay)
  365. {
  366. /*
  367. * In shadow register mode, the T11 settings are stored in
  368. * registers in the core, which are updated by the DQS_ENA
  369. * signals. Not issuing the SCC_MGR_UPD command allows us to
  370. * save lots of rank switching overhead, by calling
  371. * select_shadow_regs_for_update with update_scan_chains
  372. * set to 0.
  373. */
  374. scc_mgr_set_all_ranks(SCC_MGR_DQS_EN_DELAY_OFFSET,
  375. read_group, delay, 1);
  376. writel(0, &sdr_scc_mgr->update);
  377. }
  378. /**
  379. * scc_mgr_set_oct_out1_delay() - Set OCT output delay
  380. * @write_group: Write group
  381. * @delay: Delay value
  382. *
  383. * This function sets the OCT output delay in SCC manager.
  384. */
  385. static void scc_mgr_set_oct_out1_delay(const u32 write_group, const u32 delay)
  386. {
  387. const int ratio = RW_MGR_MEM_IF_READ_DQS_WIDTH /
  388. RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
  389. const int base = write_group * ratio;
  390. int i;
  391. /*
  392. * Load the setting in the SCC manager
  393. * Although OCT affects only write data, the OCT delay is controlled
  394. * by the DQS logic block which is instantiated once per read group.
  395. * For protocols where a write group consists of multiple read groups,
  396. * the setting must be set multiple times.
  397. */
  398. for (i = 0; i < ratio; i++)
  399. scc_mgr_set(SCC_MGR_OCT_OUT1_DELAY_OFFSET, base + i, delay);
  400. }
  401. /**
  402. * scc_mgr_set_hhp_extras() - Set HHP extras.
  403. *
  404. * Load the fixed setting in the SCC manager HHP extras.
  405. */
  406. static void scc_mgr_set_hhp_extras(void)
  407. {
  408. /*
  409. * Load the fixed setting in the SCC manager
  410. * bits: 0:0 = 1'b1 - DQS bypass
  411. * bits: 1:1 = 1'b1 - DQ bypass
  412. * bits: 4:2 = 3'b001 - rfifo_mode
  413. * bits: 6:5 = 2'b01 - rfifo clock_select
  414. * bits: 7:7 = 1'b0 - separate gating from ungating setting
  415. * bits: 8:8 = 1'b0 - separate OE from Output delay setting
  416. */
  417. const u32 value = (0 << 8) | (0 << 7) | (1 << 5) |
  418. (1 << 2) | (1 << 1) | (1 << 0);
  419. const u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS |
  420. SCC_MGR_HHP_GLOBALS_OFFSET |
  421. SCC_MGR_HHP_EXTRAS_OFFSET;
  422. debug_cond(DLEVEL == 1, "%s:%d Setting HHP Extras\n",
  423. __func__, __LINE__);
  424. writel(value, addr);
  425. debug_cond(DLEVEL == 1, "%s:%d Done Setting HHP Extras\n",
  426. __func__, __LINE__);
  427. }
  428. /**
  429. * scc_mgr_zero_all() - Zero all DQS config
  430. *
  431. * Zero all DQS config.
  432. */
  433. static void scc_mgr_zero_all(void)
  434. {
  435. int i, r;
  436. /*
  437. * USER Zero all DQS config settings, across all groups and all
  438. * shadow registers
  439. */
  440. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
  441. r += NUM_RANKS_PER_SHADOW_REG) {
  442. for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
  443. /*
  444. * The phases actually don't exist on a per-rank basis,
  445. * but there's no harm updating them several times, so
  446. * let's keep the code simple.
  447. */
  448. scc_mgr_set_dqs_bus_in_delay(i, IO_DQS_IN_RESERVE);
  449. scc_mgr_set_dqs_en_phase(i, 0);
  450. scc_mgr_set_dqs_en_delay(i, 0);
  451. }
  452. for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
  453. scc_mgr_set_dqdqs_output_phase(i, 0);
  454. /* Arria V/Cyclone V don't have out2. */
  455. scc_mgr_set_oct_out1_delay(i, IO_DQS_OUT_RESERVE);
  456. }
  457. }
  458. /* Multicast to all DQS group enables. */
  459. writel(0xff, &sdr_scc_mgr->dqs_ena);
  460. writel(0, &sdr_scc_mgr->update);
  461. }
  462. /**
  463. * scc_set_bypass_mode() - Set bypass mode and trigger SCC update
  464. * @write_group: Write group
  465. *
  466. * Set bypass mode and trigger SCC update.
  467. */
  468. static void scc_set_bypass_mode(const u32 write_group)
  469. {
  470. /* Multicast to all DQ enables. */
  471. writel(0xff, &sdr_scc_mgr->dq_ena);
  472. writel(0xff, &sdr_scc_mgr->dm_ena);
  473. /* Update current DQS IO enable. */
  474. writel(0, &sdr_scc_mgr->dqs_io_ena);
  475. /* Update the DQS logic. */
  476. writel(write_group, &sdr_scc_mgr->dqs_ena);
  477. /* Hit update. */
  478. writel(0, &sdr_scc_mgr->update);
  479. }
  480. /**
  481. * scc_mgr_load_dqs_for_write_group() - Load DQS settings for Write Group
  482. * @write_group: Write group
  483. *
  484. * Load DQS settings for Write Group, do not trigger SCC update.
  485. */
  486. static void scc_mgr_load_dqs_for_write_group(const u32 write_group)
  487. {
  488. const int ratio = RW_MGR_MEM_IF_READ_DQS_WIDTH /
  489. RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
  490. const int base = write_group * ratio;
  491. int i;
  492. /*
  493. * Load the setting in the SCC manager
  494. * Although OCT affects only write data, the OCT delay is controlled
  495. * by the DQS logic block which is instantiated once per read group.
  496. * For protocols where a write group consists of multiple read groups,
  497. * the setting must be set multiple times.
  498. */
  499. for (i = 0; i < ratio; i++)
  500. writel(base + i, &sdr_scc_mgr->dqs_ena);
  501. }
  502. /**
  503. * scc_mgr_zero_group() - Zero all configs for a group
  504. *
  505. * Zero DQ, DM, DQS and OCT configs for a group.
  506. */
  507. static void scc_mgr_zero_group(const u32 write_group, const int out_only)
  508. {
  509. int i, r;
  510. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
  511. r += NUM_RANKS_PER_SHADOW_REG) {
  512. /* Zero all DQ config settings. */
  513. for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
  514. scc_mgr_set_dq_out1_delay(i, 0);
  515. if (!out_only)
  516. scc_mgr_set_dq_in_delay(i, 0);
  517. }
  518. /* Multicast to all DQ enables. */
  519. writel(0xff, &sdr_scc_mgr->dq_ena);
  520. /* Zero all DM config settings. */
  521. for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++)
  522. scc_mgr_set_dm_out1_delay(i, 0);
  523. /* Multicast to all DM enables. */
  524. writel(0xff, &sdr_scc_mgr->dm_ena);
  525. /* Zero all DQS IO settings. */
  526. if (!out_only)
  527. scc_mgr_set_dqs_io_in_delay(0);
  528. /* Arria V/Cyclone V don't have out2. */
  529. scc_mgr_set_dqs_out1_delay(IO_DQS_OUT_RESERVE);
  530. scc_mgr_set_oct_out1_delay(write_group, IO_DQS_OUT_RESERVE);
  531. scc_mgr_load_dqs_for_write_group(write_group);
  532. /* Multicast to all DQS IO enables (only 1 in total). */
  533. writel(0, &sdr_scc_mgr->dqs_io_ena);
  534. /* Hit update to zero everything. */
  535. writel(0, &sdr_scc_mgr->update);
  536. }
  537. }
  538. /*
  539. * apply and load a particular input delay for the DQ pins in a group
  540. * group_bgn is the index of the first dq pin (in the write group)
  541. */
  542. static void scc_mgr_apply_group_dq_in_delay(uint32_t group_bgn, uint32_t delay)
  543. {
  544. uint32_t i, p;
  545. for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
  546. scc_mgr_set_dq_in_delay(p, delay);
  547. scc_mgr_load_dq(p);
  548. }
  549. }
  550. /**
  551. * scc_mgr_apply_group_dq_out1_delay() - Apply and load an output delay for the DQ pins in a group
  552. * @delay: Delay value
  553. *
  554. * Apply and load a particular output delay for the DQ pins in a group.
  555. */
  556. static void scc_mgr_apply_group_dq_out1_delay(const u32 delay)
  557. {
  558. int i;
  559. for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
  560. scc_mgr_set_dq_out1_delay(i, delay);
  561. scc_mgr_load_dq(i);
  562. }
  563. }
  564. /* apply and load a particular output delay for the DM pins in a group */
  565. static void scc_mgr_apply_group_dm_out1_delay(uint32_t delay1)
  566. {
  567. uint32_t i;
  568. for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
  569. scc_mgr_set_dm_out1_delay(i, delay1);
  570. scc_mgr_load_dm(i);
  571. }
  572. }
  573. /* apply and load delay on both DQS and OCT out1 */
  574. static void scc_mgr_apply_group_dqs_io_and_oct_out1(uint32_t write_group,
  575. uint32_t delay)
  576. {
  577. scc_mgr_set_dqs_out1_delay(delay);
  578. scc_mgr_load_dqs_io();
  579. scc_mgr_set_oct_out1_delay(write_group, delay);
  580. scc_mgr_load_dqs_for_write_group(write_group);
  581. }
  582. /**
  583. * scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output side: DQ, DM, DQS, OCT
  584. * @write_group: Write group
  585. * @delay: Delay value
  586. *
  587. * Apply a delay to the entire output side: DQ, DM, DQS, OCT.
  588. */
  589. static void scc_mgr_apply_group_all_out_delay_add(const u32 write_group,
  590. const u32 delay)
  591. {
  592. u32 i, new_delay;
  593. /* DQ shift */
  594. for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++)
  595. scc_mgr_load_dq(i);
  596. /* DM shift */
  597. for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++)
  598. scc_mgr_load_dm(i);
  599. /* DQS shift */
  600. new_delay = READ_SCC_DQS_IO_OUT2_DELAY + delay;
  601. if (new_delay > IO_IO_OUT2_DELAY_MAX) {
  602. debug_cond(DLEVEL == 1,
  603. "%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n",
  604. __func__, __LINE__, write_group, delay, new_delay,
  605. IO_IO_OUT2_DELAY_MAX,
  606. new_delay - IO_IO_OUT2_DELAY_MAX);
  607. new_delay -= IO_IO_OUT2_DELAY_MAX;
  608. scc_mgr_set_dqs_out1_delay(new_delay);
  609. }
  610. scc_mgr_load_dqs_io();
  611. /* OCT shift */
  612. new_delay = READ_SCC_OCT_OUT2_DELAY + delay;
  613. if (new_delay > IO_IO_OUT2_DELAY_MAX) {
  614. debug_cond(DLEVEL == 1,
  615. "%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n",
  616. __func__, __LINE__, write_group, delay,
  617. new_delay, IO_IO_OUT2_DELAY_MAX,
  618. new_delay - IO_IO_OUT2_DELAY_MAX);
  619. new_delay -= IO_IO_OUT2_DELAY_MAX;
  620. scc_mgr_set_oct_out1_delay(write_group, new_delay);
  621. }
  622. scc_mgr_load_dqs_for_write_group(write_group);
  623. }
  624. /**
  625. * scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output side to all ranks
  626. * @write_group: Write group
  627. * @delay: Delay value
  628. *
  629. * Apply a delay to the entire output side (DQ, DM, DQS, OCT) to all ranks.
  630. */
  631. static void
  632. scc_mgr_apply_group_all_out_delay_add_all_ranks(const u32 write_group,
  633. const u32 delay)
  634. {
  635. int r;
  636. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
  637. r += NUM_RANKS_PER_SHADOW_REG) {
  638. scc_mgr_apply_group_all_out_delay_add(write_group, delay);
  639. writel(0, &sdr_scc_mgr->update);
  640. }
  641. }
  642. /* optimization used to recover some slots in ddr3 inst_rom */
  643. /* could be applied to other protocols if we wanted to */
  644. static void set_jump_as_return(void)
  645. {
  646. /*
  647. * to save space, we replace return with jump to special shared
  648. * RETURN instruction so we set the counter to large value so that
  649. * we always jump
  650. */
  651. writel(0xff, &sdr_rw_load_mgr_regs->load_cntr0);
  652. writel(RW_MGR_RETURN, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  653. }
  654. /*
  655. * should always use constants as argument to ensure all computations are
  656. * performed at compile time
  657. */
  658. static void delay_for_n_mem_clocks(const uint32_t clocks)
  659. {
  660. uint32_t afi_clocks;
  661. uint8_t inner = 0;
  662. uint8_t outer = 0;
  663. uint16_t c_loop = 0;
  664. debug("%s:%d: clocks=%u ... start\n", __func__, __LINE__, clocks);
  665. afi_clocks = (clocks + AFI_RATE_RATIO-1) / AFI_RATE_RATIO;
  666. /* scale (rounding up) to get afi clocks */
  667. /*
  668. * Note, we don't bother accounting for being off a little bit
  669. * because of a few extra instructions in outer loops
  670. * Note, the loops have a test at the end, and do the test before
  671. * the decrement, and so always perform the loop
  672. * 1 time more than the counter value
  673. */
  674. if (afi_clocks == 0) {
  675. ;
  676. } else if (afi_clocks <= 0x100) {
  677. inner = afi_clocks-1;
  678. outer = 0;
  679. c_loop = 0;
  680. } else if (afi_clocks <= 0x10000) {
  681. inner = 0xff;
  682. outer = (afi_clocks-1) >> 8;
  683. c_loop = 0;
  684. } else {
  685. inner = 0xff;
  686. outer = 0xff;
  687. c_loop = (afi_clocks-1) >> 16;
  688. }
  689. /*
  690. * rom instructions are structured as follows:
  691. *
  692. * IDLE_LOOP2: jnz cntr0, TARGET_A
  693. * IDLE_LOOP1: jnz cntr1, TARGET_B
  694. * return
  695. *
  696. * so, when doing nested loops, TARGET_A is set to IDLE_LOOP2, and
  697. * TARGET_B is set to IDLE_LOOP2 as well
  698. *
  699. * if we have no outer loop, though, then we can use IDLE_LOOP1 only,
  700. * and set TARGET_B to IDLE_LOOP1 and we skip IDLE_LOOP2 entirely
  701. *
  702. * a little confusing, but it helps save precious space in the inst_rom
  703. * and sequencer rom and keeps the delays more accurate and reduces
  704. * overhead
  705. */
  706. if (afi_clocks <= 0x100) {
  707. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
  708. &sdr_rw_load_mgr_regs->load_cntr1);
  709. writel(RW_MGR_IDLE_LOOP1,
  710. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  711. writel(RW_MGR_IDLE_LOOP1, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  712. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  713. } else {
  714. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
  715. &sdr_rw_load_mgr_regs->load_cntr0);
  716. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(outer),
  717. &sdr_rw_load_mgr_regs->load_cntr1);
  718. writel(RW_MGR_IDLE_LOOP2,
  719. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  720. writel(RW_MGR_IDLE_LOOP2,
  721. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  722. /* hack to get around compiler not being smart enough */
  723. if (afi_clocks <= 0x10000) {
  724. /* only need to run once */
  725. writel(RW_MGR_IDLE_LOOP2, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  726. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  727. } else {
  728. do {
  729. writel(RW_MGR_IDLE_LOOP2,
  730. SDR_PHYGRP_RWMGRGRP_ADDRESS |
  731. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  732. } while (c_loop-- != 0);
  733. }
  734. }
  735. debug("%s:%d clocks=%u ... end\n", __func__, __LINE__, clocks);
  736. }
  737. /**
  738. * rw_mgr_mem_init_load_regs() - Load instruction registers
  739. * @cntr0: Counter 0 value
  740. * @cntr1: Counter 1 value
  741. * @cntr2: Counter 2 value
  742. * @jump: Jump instruction value
  743. *
  744. * Load instruction registers.
  745. */
  746. static void rw_mgr_mem_init_load_regs(u32 cntr0, u32 cntr1, u32 cntr2, u32 jump)
  747. {
  748. uint32_t grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  749. RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  750. /* Load counters */
  751. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr0),
  752. &sdr_rw_load_mgr_regs->load_cntr0);
  753. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr1),
  754. &sdr_rw_load_mgr_regs->load_cntr1);
  755. writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr2),
  756. &sdr_rw_load_mgr_regs->load_cntr2);
  757. /* Load jump address */
  758. writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  759. writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  760. writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  761. /* Execute count instruction */
  762. writel(jump, grpaddr);
  763. }
  764. /**
  765. * rw_mgr_mem_load_user() - Load user calibration values
  766. * @fin1: Final instruction 1
  767. * @fin2: Final instruction 2
  768. * @precharge: If 1, precharge the banks at the end
  769. *
  770. * Load user calibration values and optionally precharge the banks.
  771. */
  772. static void rw_mgr_mem_load_user(const u32 fin1, const u32 fin2,
  773. const int precharge)
  774. {
  775. u32 grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
  776. RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  777. u32 r;
  778. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
  779. if (param->skip_ranks[r]) {
  780. /* request to skip the rank */
  781. continue;
  782. }
  783. /* set rank */
  784. set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
  785. /* precharge all banks ... */
  786. if (precharge)
  787. writel(RW_MGR_PRECHARGE_ALL, grpaddr);
  788. /*
  789. * USER Use Mirror-ed commands for odd ranks if address
  790. * mirrorring is on
  791. */
  792. if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) {
  793. set_jump_as_return();
  794. writel(RW_MGR_MRS2_MIRR, grpaddr);
  795. delay_for_n_mem_clocks(4);
  796. set_jump_as_return();
  797. writel(RW_MGR_MRS3_MIRR, grpaddr);
  798. delay_for_n_mem_clocks(4);
  799. set_jump_as_return();
  800. writel(RW_MGR_MRS1_MIRR, grpaddr);
  801. delay_for_n_mem_clocks(4);
  802. set_jump_as_return();
  803. writel(fin1, grpaddr);
  804. } else {
  805. set_jump_as_return();
  806. writel(RW_MGR_MRS2, grpaddr);
  807. delay_for_n_mem_clocks(4);
  808. set_jump_as_return();
  809. writel(RW_MGR_MRS3, grpaddr);
  810. delay_for_n_mem_clocks(4);
  811. set_jump_as_return();
  812. writel(RW_MGR_MRS1, grpaddr);
  813. set_jump_as_return();
  814. writel(fin2, grpaddr);
  815. }
  816. if (precharge)
  817. continue;
  818. set_jump_as_return();
  819. writel(RW_MGR_ZQCL, grpaddr);
  820. /* tZQinit = tDLLK = 512 ck cycles */
  821. delay_for_n_mem_clocks(512);
  822. }
  823. }
  824. static void rw_mgr_mem_initialize(void)
  825. {
  826. debug("%s:%d\n", __func__, __LINE__);
  827. /* The reset / cke part of initialization is broadcasted to all ranks */
  828. writel(RW_MGR_RANK_ALL, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  829. RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
  830. /*
  831. * Here's how you load register for a loop
  832. * Counters are located @ 0x800
  833. * Jump address are located @ 0xC00
  834. * For both, registers 0 to 3 are selected using bits 3 and 2, like
  835. * in 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C
  836. * I know this ain't pretty, but Avalon bus throws away the 2 least
  837. * significant bits
  838. */
  839. /* start with memory RESET activated */
  840. /* tINIT = 200us */
  841. /*
  842. * 200us @ 266MHz (3.75 ns) ~ 54000 clock cycles
  843. * If a and b are the number of iteration in 2 nested loops
  844. * it takes the following number of cycles to complete the operation:
  845. * number_of_cycles = ((2 + n) * a + 2) * b
  846. * where n is the number of instruction in the inner loop
  847. * One possible solution is n = 0 , a = 256 , b = 106 => a = FF,
  848. * b = 6A
  849. */
  850. rw_mgr_mem_init_load_regs(SEQ_TINIT_CNTR0_VAL, SEQ_TINIT_CNTR1_VAL,
  851. SEQ_TINIT_CNTR2_VAL,
  852. RW_MGR_INIT_RESET_0_CKE_0);
  853. /* indicate that memory is stable */
  854. writel(1, &phy_mgr_cfg->reset_mem_stbl);
  855. /*
  856. * transition the RESET to high
  857. * Wait for 500us
  858. */
  859. /*
  860. * 500us @ 266MHz (3.75 ns) ~ 134000 clock cycles
  861. * If a and b are the number of iteration in 2 nested loops
  862. * it takes the following number of cycles to complete the operation
  863. * number_of_cycles = ((2 + n) * a + 2) * b
  864. * where n is the number of instruction in the inner loop
  865. * One possible solution is n = 2 , a = 131 , b = 256 => a = 83,
  866. * b = FF
  867. */
  868. rw_mgr_mem_init_load_regs(SEQ_TRESET_CNTR0_VAL, SEQ_TRESET_CNTR1_VAL,
  869. SEQ_TRESET_CNTR2_VAL,
  870. RW_MGR_INIT_RESET_1_CKE_0);
  871. /* bring up clock enable */
  872. /* tXRP < 250 ck cycles */
  873. delay_for_n_mem_clocks(250);
  874. rw_mgr_mem_load_user(RW_MGR_MRS0_DLL_RESET_MIRR, RW_MGR_MRS0_DLL_RESET,
  875. 0);
  876. }
  877. /*
  878. * At the end of calibration we have to program the user settings in, and
  879. * USER hand off the memory to the user.
  880. */
  881. static void rw_mgr_mem_handoff(void)
  882. {
  883. rw_mgr_mem_load_user(RW_MGR_MRS0_USER_MIRR, RW_MGR_MRS0_USER, 1);
  884. /*
  885. * USER need to wait tMOD (12CK or 15ns) time before issuing
  886. * other commands, but we will have plenty of NIOS cycles before
  887. * actual handoff so its okay.
  888. */
  889. }
  890. /*
  891. * performs a guaranteed read on the patterns we are going to use during a
  892. * read test to ensure memory works
  893. */
  894. static uint32_t rw_mgr_mem_calibrate_read_test_patterns(uint32_t rank_bgn,
  895. uint32_t group, uint32_t num_tries, uint32_t *bit_chk,
  896. uint32_t all_ranks)
  897. {
  898. uint32_t r, vg;
  899. uint32_t correct_mask_vg;
  900. uint32_t tmp_bit_chk;
  901. uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
  902. (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
  903. uint32_t addr;
  904. uint32_t base_rw_mgr;
  905. *bit_chk = param->read_correct_mask;
  906. correct_mask_vg = param->read_correct_mask_vg;
  907. for (r = rank_bgn; r < rank_end; r++) {
  908. if (param->skip_ranks[r])
  909. /* request to skip the rank */
  910. continue;
  911. /* set rank */
  912. set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
  913. /* Load up a constant bursts of read commands */
  914. writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
  915. writel(RW_MGR_GUARANTEED_READ,
  916. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  917. writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
  918. writel(RW_MGR_GUARANTEED_READ_CONT,
  919. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  920. tmp_bit_chk = 0;
  921. for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) {
  922. /* reset the fifos to get pointers to known state */
  923. writel(0, &phy_mgr_cmd->fifo_reset);
  924. writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  925. RW_MGR_RESET_READ_DATAPATH_OFFSET);
  926. tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS
  927. / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS);
  928. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  929. writel(RW_MGR_GUARANTEED_READ, addr +
  930. ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS +
  931. vg) << 2));
  932. base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
  933. tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & (~base_rw_mgr));
  934. if (vg == 0)
  935. break;
  936. }
  937. *bit_chk &= tmp_bit_chk;
  938. }
  939. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  940. writel(RW_MGR_CLEAR_DQS_ENABLE, addr + (group << 2));
  941. set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
  942. debug_cond(DLEVEL == 1, "%s:%d test_load_patterns(%u,ALL) => (%u == %u) =>\
  943. %lu\n", __func__, __LINE__, group, *bit_chk, param->read_correct_mask,
  944. (long unsigned int)(*bit_chk == param->read_correct_mask));
  945. return *bit_chk == param->read_correct_mask;
  946. }
  947. static uint32_t rw_mgr_mem_calibrate_read_test_patterns_all_ranks
  948. (uint32_t group, uint32_t num_tries, uint32_t *bit_chk)
  949. {
  950. return rw_mgr_mem_calibrate_read_test_patterns(0, group,
  951. num_tries, bit_chk, 1);
  952. }
  953. /* load up the patterns we are going to use during a read test */
  954. static void rw_mgr_mem_calibrate_read_load_patterns(uint32_t rank_bgn,
  955. uint32_t all_ranks)
  956. {
  957. uint32_t r;
  958. uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
  959. (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
  960. debug("%s:%d\n", __func__, __LINE__);
  961. for (r = rank_bgn; r < rank_end; r++) {
  962. if (param->skip_ranks[r])
  963. /* request to skip the rank */
  964. continue;
  965. /* set rank */
  966. set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
  967. /* Load up a constant bursts */
  968. writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
  969. writel(RW_MGR_GUARANTEED_WRITE_WAIT0,
  970. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  971. writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
  972. writel(RW_MGR_GUARANTEED_WRITE_WAIT1,
  973. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  974. writel(0x04, &sdr_rw_load_mgr_regs->load_cntr2);
  975. writel(RW_MGR_GUARANTEED_WRITE_WAIT2,
  976. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  977. writel(0x04, &sdr_rw_load_mgr_regs->load_cntr3);
  978. writel(RW_MGR_GUARANTEED_WRITE_WAIT3,
  979. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  980. writel(RW_MGR_GUARANTEED_WRITE, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  981. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  982. }
  983. set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
  984. }
  985. /*
  986. * try a read and see if it returns correct data back. has dummy reads
  987. * inserted into the mix used to align dqs enable. has more thorough checks
  988. * than the regular read test.
  989. */
  990. static uint32_t rw_mgr_mem_calibrate_read_test(uint32_t rank_bgn, uint32_t group,
  991. uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
  992. uint32_t all_groups, uint32_t all_ranks)
  993. {
  994. uint32_t r, vg;
  995. uint32_t correct_mask_vg;
  996. uint32_t tmp_bit_chk;
  997. uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
  998. (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
  999. uint32_t addr;
  1000. uint32_t base_rw_mgr;
  1001. *bit_chk = param->read_correct_mask;
  1002. correct_mask_vg = param->read_correct_mask_vg;
  1003. uint32_t quick_read_mode = (((STATIC_CALIB_STEPS) &
  1004. CALIB_SKIP_DELAY_SWEEPS) && ENABLE_SUPER_QUICK_CALIBRATION);
  1005. for (r = rank_bgn; r < rank_end; r++) {
  1006. if (param->skip_ranks[r])
  1007. /* request to skip the rank */
  1008. continue;
  1009. /* set rank */
  1010. set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
  1011. writel(0x10, &sdr_rw_load_mgr_regs->load_cntr1);
  1012. writel(RW_MGR_READ_B2B_WAIT1,
  1013. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  1014. writel(0x10, &sdr_rw_load_mgr_regs->load_cntr2);
  1015. writel(RW_MGR_READ_B2B_WAIT2,
  1016. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  1017. if (quick_read_mode)
  1018. writel(0x1, &sdr_rw_load_mgr_regs->load_cntr0);
  1019. /* need at least two (1+1) reads to capture failures */
  1020. else if (all_groups)
  1021. writel(0x06, &sdr_rw_load_mgr_regs->load_cntr0);
  1022. else
  1023. writel(0x32, &sdr_rw_load_mgr_regs->load_cntr0);
  1024. writel(RW_MGR_READ_B2B,
  1025. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  1026. if (all_groups)
  1027. writel(RW_MGR_MEM_IF_READ_DQS_WIDTH *
  1028. RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS - 1,
  1029. &sdr_rw_load_mgr_regs->load_cntr3);
  1030. else
  1031. writel(0x0, &sdr_rw_load_mgr_regs->load_cntr3);
  1032. writel(RW_MGR_READ_B2B,
  1033. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  1034. tmp_bit_chk = 0;
  1035. for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) {
  1036. /* reset the fifos to get pointers to known state */
  1037. writel(0, &phy_mgr_cmd->fifo_reset);
  1038. writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  1039. RW_MGR_RESET_READ_DATAPATH_OFFSET);
  1040. tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS
  1041. / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS);
  1042. if (all_groups)
  1043. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_ALL_GROUPS_OFFSET;
  1044. else
  1045. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  1046. writel(RW_MGR_READ_B2B, addr +
  1047. ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS +
  1048. vg) << 2));
  1049. base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
  1050. tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
  1051. if (vg == 0)
  1052. break;
  1053. }
  1054. *bit_chk &= tmp_bit_chk;
  1055. }
  1056. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  1057. writel(RW_MGR_CLEAR_DQS_ENABLE, addr + (group << 2));
  1058. if (all_correct) {
  1059. set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
  1060. debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ALL,%u) =>\
  1061. (%u == %u) => %lu", __func__, __LINE__, group,
  1062. all_groups, *bit_chk, param->read_correct_mask,
  1063. (long unsigned int)(*bit_chk ==
  1064. param->read_correct_mask));
  1065. return *bit_chk == param->read_correct_mask;
  1066. } else {
  1067. set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
  1068. debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ONE,%u) =>\
  1069. (%u != %lu) => %lu\n", __func__, __LINE__,
  1070. group, all_groups, *bit_chk, (long unsigned int)0,
  1071. (long unsigned int)(*bit_chk != 0x00));
  1072. return *bit_chk != 0x00;
  1073. }
  1074. }
  1075. static uint32_t rw_mgr_mem_calibrate_read_test_all_ranks(uint32_t group,
  1076. uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
  1077. uint32_t all_groups)
  1078. {
  1079. return rw_mgr_mem_calibrate_read_test(0, group, num_tries, all_correct,
  1080. bit_chk, all_groups, 1);
  1081. }
  1082. static void rw_mgr_incr_vfifo(uint32_t grp, uint32_t *v)
  1083. {
  1084. writel(grp, &phy_mgr_cmd->inc_vfifo_hard_phy);
  1085. (*v)++;
  1086. }
  1087. static void rw_mgr_decr_vfifo(uint32_t grp, uint32_t *v)
  1088. {
  1089. uint32_t i;
  1090. for (i = 0; i < VFIFO_SIZE-1; i++)
  1091. rw_mgr_incr_vfifo(grp, v);
  1092. }
  1093. static int find_vfifo_read(uint32_t grp, uint32_t *bit_chk)
  1094. {
  1095. uint32_t v;
  1096. uint32_t fail_cnt = 0;
  1097. uint32_t test_status;
  1098. for (v = 0; v < VFIFO_SIZE; ) {
  1099. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo %u\n",
  1100. __func__, __LINE__, v);
  1101. test_status = rw_mgr_mem_calibrate_read_test_all_ranks
  1102. (grp, 1, PASS_ONE_BIT, bit_chk, 0);
  1103. if (!test_status) {
  1104. fail_cnt++;
  1105. if (fail_cnt == 2)
  1106. break;
  1107. }
  1108. /* fiddle with FIFO */
  1109. rw_mgr_incr_vfifo(grp, &v);
  1110. }
  1111. if (v >= VFIFO_SIZE) {
  1112. /* no failing read found!! Something must have gone wrong */
  1113. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo failed\n",
  1114. __func__, __LINE__);
  1115. return 0;
  1116. } else {
  1117. return v;
  1118. }
  1119. }
  1120. static int find_working_phase(uint32_t *grp, uint32_t *bit_chk,
  1121. uint32_t dtaps_per_ptap, uint32_t *work_bgn,
  1122. uint32_t *v, uint32_t *d, uint32_t *p,
  1123. uint32_t *i, uint32_t *max_working_cnt)
  1124. {
  1125. uint32_t found_begin = 0;
  1126. uint32_t tmp_delay = 0;
  1127. uint32_t test_status;
  1128. for (*d = 0; *d <= dtaps_per_ptap; (*d)++, tmp_delay +=
  1129. IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
  1130. *work_bgn = tmp_delay;
  1131. scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
  1132. for (*i = 0; *i < VFIFO_SIZE; (*i)++) {
  1133. for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_bgn +=
  1134. IO_DELAY_PER_OPA_TAP) {
  1135. scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
  1136. test_status =
  1137. rw_mgr_mem_calibrate_read_test_all_ranks
  1138. (*grp, 1, PASS_ONE_BIT, bit_chk, 0);
  1139. if (test_status) {
  1140. *max_working_cnt = 1;
  1141. found_begin = 1;
  1142. break;
  1143. }
  1144. }
  1145. if (found_begin)
  1146. break;
  1147. if (*p > IO_DQS_EN_PHASE_MAX)
  1148. /* fiddle with FIFO */
  1149. rw_mgr_incr_vfifo(*grp, v);
  1150. }
  1151. if (found_begin)
  1152. break;
  1153. }
  1154. if (*i >= VFIFO_SIZE) {
  1155. /* cannot find working solution */
  1156. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: no vfifo/\
  1157. ptap/dtap\n", __func__, __LINE__);
  1158. return 0;
  1159. } else {
  1160. return 1;
  1161. }
  1162. }
  1163. static void sdr_backup_phase(uint32_t *grp, uint32_t *bit_chk,
  1164. uint32_t *work_bgn, uint32_t *v, uint32_t *d,
  1165. uint32_t *p, uint32_t *max_working_cnt)
  1166. {
  1167. uint32_t found_begin = 0;
  1168. uint32_t tmp_delay;
  1169. /* Special case code for backing up a phase */
  1170. if (*p == 0) {
  1171. *p = IO_DQS_EN_PHASE_MAX;
  1172. rw_mgr_decr_vfifo(*grp, v);
  1173. } else {
  1174. (*p)--;
  1175. }
  1176. tmp_delay = *work_bgn - IO_DELAY_PER_OPA_TAP;
  1177. scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
  1178. for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_bgn;
  1179. (*d)++, tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
  1180. scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
  1181. if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1,
  1182. PASS_ONE_BIT,
  1183. bit_chk, 0)) {
  1184. found_begin = 1;
  1185. *work_bgn = tmp_delay;
  1186. break;
  1187. }
  1188. }
  1189. /* We have found a working dtap before the ptap found above */
  1190. if (found_begin == 1)
  1191. (*max_working_cnt)++;
  1192. /*
  1193. * Restore VFIFO to old state before we decremented it
  1194. * (if needed).
  1195. */
  1196. (*p)++;
  1197. if (*p > IO_DQS_EN_PHASE_MAX) {
  1198. *p = 0;
  1199. rw_mgr_incr_vfifo(*grp, v);
  1200. }
  1201. scc_mgr_set_dqs_en_delay_all_ranks(*grp, 0);
  1202. }
  1203. static int sdr_nonworking_phase(uint32_t *grp, uint32_t *bit_chk,
  1204. uint32_t *work_bgn, uint32_t *v, uint32_t *d,
  1205. uint32_t *p, uint32_t *i, uint32_t *max_working_cnt,
  1206. uint32_t *work_end)
  1207. {
  1208. uint32_t found_end = 0;
  1209. (*p)++;
  1210. *work_end += IO_DELAY_PER_OPA_TAP;
  1211. if (*p > IO_DQS_EN_PHASE_MAX) {
  1212. /* fiddle with FIFO */
  1213. *p = 0;
  1214. rw_mgr_incr_vfifo(*grp, v);
  1215. }
  1216. for (; *i < VFIFO_SIZE + 1; (*i)++) {
  1217. for (; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_end
  1218. += IO_DELAY_PER_OPA_TAP) {
  1219. scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
  1220. if (!rw_mgr_mem_calibrate_read_test_all_ranks
  1221. (*grp, 1, PASS_ONE_BIT, bit_chk, 0)) {
  1222. found_end = 1;
  1223. break;
  1224. } else {
  1225. (*max_working_cnt)++;
  1226. }
  1227. }
  1228. if (found_end)
  1229. break;
  1230. if (*p > IO_DQS_EN_PHASE_MAX) {
  1231. /* fiddle with FIFO */
  1232. rw_mgr_incr_vfifo(*grp, v);
  1233. *p = 0;
  1234. }
  1235. }
  1236. if (*i >= VFIFO_SIZE + 1) {
  1237. /* cannot see edge of failing read */
  1238. debug_cond(DLEVEL == 2, "%s:%d sdr_nonworking_phase: end:\
  1239. failed\n", __func__, __LINE__);
  1240. return 0;
  1241. } else {
  1242. return 1;
  1243. }
  1244. }
  1245. static int sdr_find_window_centre(uint32_t *grp, uint32_t *bit_chk,
  1246. uint32_t *work_bgn, uint32_t *v, uint32_t *d,
  1247. uint32_t *p, uint32_t *work_mid,
  1248. uint32_t *work_end)
  1249. {
  1250. int i;
  1251. int tmp_delay = 0;
  1252. *work_mid = (*work_bgn + *work_end) / 2;
  1253. debug_cond(DLEVEL == 2, "work_bgn=%d work_end=%d work_mid=%d\n",
  1254. *work_bgn, *work_end, *work_mid);
  1255. /* Get the middle delay to be less than a VFIFO delay */
  1256. for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX;
  1257. (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP)
  1258. ;
  1259. debug_cond(DLEVEL == 2, "vfifo ptap delay %d\n", tmp_delay);
  1260. while (*work_mid > tmp_delay)
  1261. *work_mid -= tmp_delay;
  1262. debug_cond(DLEVEL == 2, "new work_mid %d\n", *work_mid);
  1263. tmp_delay = 0;
  1264. for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX && tmp_delay < *work_mid;
  1265. (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP)
  1266. ;
  1267. tmp_delay -= IO_DELAY_PER_OPA_TAP;
  1268. debug_cond(DLEVEL == 2, "new p %d, tmp_delay=%d\n", (*p) - 1, tmp_delay);
  1269. for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_mid; (*d)++,
  1270. tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP)
  1271. ;
  1272. debug_cond(DLEVEL == 2, "new d %d, tmp_delay=%d\n", *d, tmp_delay);
  1273. scc_mgr_set_dqs_en_phase_all_ranks(*grp, (*p) - 1);
  1274. scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
  1275. /*
  1276. * push vfifo until we can successfully calibrate. We can do this
  1277. * because the largest possible margin in 1 VFIFO cycle.
  1278. */
  1279. for (i = 0; i < VFIFO_SIZE; i++) {
  1280. debug_cond(DLEVEL == 2, "find_dqs_en_phase: center: vfifo=%u\n",
  1281. *v);
  1282. if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1,
  1283. PASS_ONE_BIT,
  1284. bit_chk, 0)) {
  1285. break;
  1286. }
  1287. /* fiddle with FIFO */
  1288. rw_mgr_incr_vfifo(*grp, v);
  1289. }
  1290. if (i >= VFIFO_SIZE) {
  1291. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center: \
  1292. failed\n", __func__, __LINE__);
  1293. return 0;
  1294. } else {
  1295. return 1;
  1296. }
  1297. }
  1298. /* find a good dqs enable to use */
  1299. static uint32_t rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(uint32_t grp)
  1300. {
  1301. uint32_t v, d, p, i;
  1302. uint32_t max_working_cnt;
  1303. uint32_t bit_chk;
  1304. uint32_t dtaps_per_ptap;
  1305. uint32_t work_bgn, work_mid, work_end;
  1306. uint32_t found_passing_read, found_failing_read, initial_failing_dtap;
  1307. debug("%s:%d %u\n", __func__, __LINE__, grp);
  1308. reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
  1309. scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);
  1310. scc_mgr_set_dqs_en_phase_all_ranks(grp, 0);
  1311. /* ************************************************************** */
  1312. /* * Step 0 : Determine number of delay taps for each phase tap * */
  1313. dtaps_per_ptap = IO_DELAY_PER_OPA_TAP/IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
  1314. /* ********************************************************* */
  1315. /* * Step 1 : First push vfifo until we get a failing read * */
  1316. v = find_vfifo_read(grp, &bit_chk);
  1317. max_working_cnt = 0;
  1318. /* ******************************************************** */
  1319. /* * step 2: find first working phase, increment in ptaps * */
  1320. work_bgn = 0;
  1321. if (find_working_phase(&grp, &bit_chk, dtaps_per_ptap, &work_bgn, &v, &d,
  1322. &p, &i, &max_working_cnt) == 0)
  1323. return 0;
  1324. work_end = work_bgn;
  1325. /*
  1326. * If d is 0 then the working window covers a phase tap and
  1327. * we can follow the old procedure otherwise, we've found the beginning,
  1328. * and we need to increment the dtaps until we find the end.
  1329. */
  1330. if (d == 0) {
  1331. /* ********************************************************* */
  1332. /* * step 3a: if we have room, back off by one and
  1333. increment in dtaps * */
  1334. sdr_backup_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p,
  1335. &max_working_cnt);
  1336. /* ********************************************************* */
  1337. /* * step 4a: go forward from working phase to non working
  1338. phase, increment in ptaps * */
  1339. if (sdr_nonworking_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p,
  1340. &i, &max_working_cnt, &work_end) == 0)
  1341. return 0;
  1342. /* ********************************************************* */
  1343. /* * step 5a: back off one from last, increment in dtaps * */
  1344. /* Special case code for backing up a phase */
  1345. if (p == 0) {
  1346. p = IO_DQS_EN_PHASE_MAX;
  1347. rw_mgr_decr_vfifo(grp, &v);
  1348. } else {
  1349. p = p - 1;
  1350. }
  1351. work_end -= IO_DELAY_PER_OPA_TAP;
  1352. scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
  1353. /* * The actual increment of dtaps is done outside of
  1354. the if/else loop to share code */
  1355. d = 0;
  1356. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p: \
  1357. vfifo=%u ptap=%u\n", __func__, __LINE__,
  1358. v, p);
  1359. } else {
  1360. /* ******************************************************* */
  1361. /* * step 3-5b: Find the right edge of the window using
  1362. delay taps * */
  1363. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase:vfifo=%u \
  1364. ptap=%u dtap=%u bgn=%u\n", __func__, __LINE__,
  1365. v, p, d, work_bgn);
  1366. work_end = work_bgn;
  1367. /* * The actual increment of dtaps is done outside of the
  1368. if/else loop to share code */
  1369. /* Only here to counterbalance a subtract later on which is
  1370. not needed if this branch of the algorithm is taken */
  1371. max_working_cnt++;
  1372. }
  1373. /* The dtap increment to find the failing edge is done here */
  1374. for (; d <= IO_DQS_EN_DELAY_MAX; d++, work_end +=
  1375. IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
  1376. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \
  1377. end-2: dtap=%u\n", __func__, __LINE__, d);
  1378. scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
  1379. if (!rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
  1380. PASS_ONE_BIT,
  1381. &bit_chk, 0)) {
  1382. break;
  1383. }
  1384. }
  1385. /* Go back to working dtap */
  1386. if (d != 0)
  1387. work_end -= IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
  1388. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p/d: vfifo=%u \
  1389. ptap=%u dtap=%u end=%u\n", __func__, __LINE__,
  1390. v, p, d-1, work_end);
  1391. if (work_end < work_bgn) {
  1392. /* nil range */
  1393. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: end-2: \
  1394. failed\n", __func__, __LINE__);
  1395. return 0;
  1396. }
  1397. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: found range [%u,%u]\n",
  1398. __func__, __LINE__, work_bgn, work_end);
  1399. /* *************************************************************** */
  1400. /*
  1401. * * We need to calculate the number of dtaps that equal a ptap
  1402. * * To do that we'll back up a ptap and re-find the edge of the
  1403. * * window using dtaps
  1404. */
  1405. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: calculate dtaps_per_ptap \
  1406. for tracking\n", __func__, __LINE__);
  1407. /* Special case code for backing up a phase */
  1408. if (p == 0) {
  1409. p = IO_DQS_EN_PHASE_MAX;
  1410. rw_mgr_decr_vfifo(grp, &v);
  1411. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \
  1412. cycle/phase: v=%u p=%u\n", __func__, __LINE__,
  1413. v, p);
  1414. } else {
  1415. p = p - 1;
  1416. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \
  1417. phase only: v=%u p=%u", __func__, __LINE__,
  1418. v, p);
  1419. }
  1420. scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
  1421. /*
  1422. * Increase dtap until we first see a passing read (in case the
  1423. * window is smaller than a ptap),
  1424. * and then a failing read to mark the edge of the window again
  1425. */
  1426. /* Find a passing read */
  1427. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find passing read\n",
  1428. __func__, __LINE__);
  1429. found_passing_read = 0;
  1430. found_failing_read = 0;
  1431. initial_failing_dtap = d;
  1432. for (; d <= IO_DQS_EN_DELAY_MAX; d++) {
  1433. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: testing \
  1434. read d=%u\n", __func__, __LINE__, d);
  1435. scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
  1436. if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
  1437. PASS_ONE_BIT,
  1438. &bit_chk, 0)) {
  1439. found_passing_read = 1;
  1440. break;
  1441. }
  1442. }
  1443. if (found_passing_read) {
  1444. /* Find a failing read */
  1445. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find failing \
  1446. read\n", __func__, __LINE__);
  1447. for (d = d + 1; d <= IO_DQS_EN_DELAY_MAX; d++) {
  1448. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \
  1449. testing read d=%u\n", __func__, __LINE__, d);
  1450. scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
  1451. if (!rw_mgr_mem_calibrate_read_test_all_ranks
  1452. (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) {
  1453. found_failing_read = 1;
  1454. break;
  1455. }
  1456. }
  1457. } else {
  1458. debug_cond(DLEVEL == 1, "%s:%d find_dqs_en_phase: failed to \
  1459. calculate dtaps", __func__, __LINE__);
  1460. debug_cond(DLEVEL == 1, "per ptap. Fall back on static value\n");
  1461. }
  1462. /*
  1463. * The dynamically calculated dtaps_per_ptap is only valid if we
  1464. * found a passing/failing read. If we didn't, it means d hit the max
  1465. * (IO_DQS_EN_DELAY_MAX). Otherwise, dtaps_per_ptap retains its
  1466. * statically calculated value.
  1467. */
  1468. if (found_passing_read && found_failing_read)
  1469. dtaps_per_ptap = d - initial_failing_dtap;
  1470. writel(dtaps_per_ptap, &sdr_reg_file->dtaps_per_ptap);
  1471. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: dtaps_per_ptap=%u \
  1472. - %u = %u", __func__, __LINE__, d,
  1473. initial_failing_dtap, dtaps_per_ptap);
  1474. /* ******************************************** */
  1475. /* * step 6: Find the centre of the window * */
  1476. if (sdr_find_window_centre(&grp, &bit_chk, &work_bgn, &v, &d, &p,
  1477. &work_mid, &work_end) == 0)
  1478. return 0;
  1479. debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center found: \
  1480. vfifo=%u ptap=%u dtap=%u\n", __func__, __LINE__,
  1481. v, p-1, d);
  1482. return 1;
  1483. }
  1484. /*
  1485. * Try rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase across different
  1486. * dq_in_delay values
  1487. */
  1488. static uint32_t
  1489. rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay
  1490. (uint32_t write_group, uint32_t read_group, uint32_t test_bgn)
  1491. {
  1492. uint32_t found;
  1493. uint32_t i;
  1494. uint32_t p;
  1495. uint32_t d;
  1496. uint32_t r;
  1497. const uint32_t delay_step = IO_IO_IN_DELAY_MAX /
  1498. (RW_MGR_MEM_DQ_PER_READ_DQS-1);
  1499. /* we start at zero, so have one less dq to devide among */
  1500. debug("%s:%d (%u,%u,%u)", __func__, __LINE__, write_group, read_group,
  1501. test_bgn);
  1502. /* try different dq_in_delays since the dq path is shorter than dqs */
  1503. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
  1504. r += NUM_RANKS_PER_SHADOW_REG) {
  1505. for (i = 0, p = test_bgn, d = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++, d += delay_step) {
  1506. debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_\
  1507. vfifo_find_dqs_", __func__, __LINE__);
  1508. debug_cond(DLEVEL == 1, "en_phase_sweep_dq_in_delay: g=%u/%u ",
  1509. write_group, read_group);
  1510. debug_cond(DLEVEL == 1, "r=%u, i=%u p=%u d=%u\n", r, i , p, d);
  1511. scc_mgr_set_dq_in_delay(p, d);
  1512. scc_mgr_load_dq(p);
  1513. }
  1514. writel(0, &sdr_scc_mgr->update);
  1515. }
  1516. found = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(read_group);
  1517. debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_vfifo_find_dqs_\
  1518. en_phase_sweep_dq", __func__, __LINE__);
  1519. debug_cond(DLEVEL == 1, "_in_delay: g=%u/%u found=%u; Reseting delay \
  1520. chain to zero\n", write_group, read_group, found);
  1521. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
  1522. r += NUM_RANKS_PER_SHADOW_REG) {
  1523. for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS;
  1524. i++, p++) {
  1525. scc_mgr_set_dq_in_delay(p, 0);
  1526. scc_mgr_load_dq(p);
  1527. }
  1528. writel(0, &sdr_scc_mgr->update);
  1529. }
  1530. return found;
  1531. }
  1532. /* per-bit deskew DQ and center */
  1533. static uint32_t rw_mgr_mem_calibrate_vfifo_center(uint32_t rank_bgn,
  1534. uint32_t write_group, uint32_t read_group, uint32_t test_bgn,
  1535. uint32_t use_read_test, uint32_t update_fom)
  1536. {
  1537. uint32_t i, p, d, min_index;
  1538. /*
  1539. * Store these as signed since there are comparisons with
  1540. * signed numbers.
  1541. */
  1542. uint32_t bit_chk;
  1543. uint32_t sticky_bit_chk;
  1544. int32_t left_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
  1545. int32_t right_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
  1546. int32_t final_dq[RW_MGR_MEM_DQ_PER_READ_DQS];
  1547. int32_t mid;
  1548. int32_t orig_mid_min, mid_min;
  1549. int32_t new_dqs, start_dqs, start_dqs_en, shift_dq, final_dqs,
  1550. final_dqs_en;
  1551. int32_t dq_margin, dqs_margin;
  1552. uint32_t stop;
  1553. uint32_t temp_dq_in_delay1, temp_dq_in_delay2;
  1554. uint32_t addr;
  1555. debug("%s:%d: %u %u", __func__, __LINE__, read_group, test_bgn);
  1556. addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_DQS_IN_DELAY_OFFSET;
  1557. start_dqs = readl(addr + (read_group << 2));
  1558. if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
  1559. start_dqs_en = readl(addr + ((read_group << 2)
  1560. - IO_DQS_EN_DELAY_OFFSET));
  1561. /* set the left and right edge of each bit to an illegal value */
  1562. /* use (IO_IO_IN_DELAY_MAX + 1) as an illegal value */
  1563. sticky_bit_chk = 0;
  1564. for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
  1565. left_edge[i] = IO_IO_IN_DELAY_MAX + 1;
  1566. right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
  1567. }
  1568. /* Search for the left edge of the window for each bit */
  1569. for (d = 0; d <= IO_IO_IN_DELAY_MAX; d++) {
  1570. scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, d);
  1571. writel(0, &sdr_scc_mgr->update);
  1572. /*
  1573. * Stop searching when the read test doesn't pass AND when
  1574. * we've seen a passing read on every bit.
  1575. */
  1576. if (use_read_test) {
  1577. stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
  1578. read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
  1579. &bit_chk, 0, 0);
  1580. } else {
  1581. rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
  1582. 0, PASS_ONE_BIT,
  1583. &bit_chk, 0);
  1584. bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
  1585. (read_group - (write_group *
  1586. RW_MGR_MEM_IF_READ_DQS_WIDTH /
  1587. RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
  1588. stop = (bit_chk == 0);
  1589. }
  1590. sticky_bit_chk = sticky_bit_chk | bit_chk;
  1591. stop = stop && (sticky_bit_chk == param->read_correct_mask);
  1592. debug_cond(DLEVEL == 2, "%s:%d vfifo_center(left): dtap=%u => %u == %u \
  1593. && %u", __func__, __LINE__, d,
  1594. sticky_bit_chk,
  1595. param->read_correct_mask, stop);
  1596. if (stop == 1) {
  1597. break;
  1598. } else {
  1599. for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
  1600. if (bit_chk & 1) {
  1601. /* Remember a passing test as the
  1602. left_edge */
  1603. left_edge[i] = d;
  1604. } else {
  1605. /* If a left edge has not been seen yet,
  1606. then a future passing test will mark
  1607. this edge as the right edge */
  1608. if (left_edge[i] ==
  1609. IO_IO_IN_DELAY_MAX + 1) {
  1610. right_edge[i] = -(d + 1);
  1611. }
  1612. }
  1613. bit_chk = bit_chk >> 1;
  1614. }
  1615. }
  1616. }
  1617. /* Reset DQ delay chains to 0 */
  1618. scc_mgr_apply_group_dq_in_delay(test_bgn, 0);
  1619. sticky_bit_chk = 0;
  1620. for (i = RW_MGR_MEM_DQ_PER_READ_DQS - 1;; i--) {
  1621. debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
  1622. %d right_edge[%u]: %d\n", __func__, __LINE__,
  1623. i, left_edge[i], i, right_edge[i]);
  1624. /*
  1625. * Check for cases where we haven't found the left edge,
  1626. * which makes our assignment of the the right edge invalid.
  1627. * Reset it to the illegal value.
  1628. */
  1629. if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) && (
  1630. right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
  1631. right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
  1632. debug_cond(DLEVEL == 2, "%s:%d vfifo_center: reset \
  1633. right_edge[%u]: %d\n", __func__, __LINE__,
  1634. i, right_edge[i]);
  1635. }
  1636. /*
  1637. * Reset sticky bit (except for bits where we have seen
  1638. * both the left and right edge).
  1639. */
  1640. sticky_bit_chk = sticky_bit_chk << 1;
  1641. if ((left_edge[i] != IO_IO_IN_DELAY_MAX + 1) &&
  1642. (right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
  1643. sticky_bit_chk = sticky_bit_chk | 1;
  1644. }
  1645. if (i == 0)
  1646. break;
  1647. }
  1648. /* Search for the right edge of the window for each bit */
  1649. for (d = 0; d <= IO_DQS_IN_DELAY_MAX - start_dqs; d++) {
  1650. scc_mgr_set_dqs_bus_in_delay(read_group, d + start_dqs);
  1651. if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
  1652. uint32_t delay = d + start_dqs_en;
  1653. if (delay > IO_DQS_EN_DELAY_MAX)
  1654. delay = IO_DQS_EN_DELAY_MAX;
  1655. scc_mgr_set_dqs_en_delay(read_group, delay);
  1656. }
  1657. scc_mgr_load_dqs(read_group);
  1658. writel(0, &sdr_scc_mgr->update);
  1659. /*
  1660. * Stop searching when the read test doesn't pass AND when
  1661. * we've seen a passing read on every bit.
  1662. */
  1663. if (use_read_test) {
  1664. stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
  1665. read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
  1666. &bit_chk, 0, 0);
  1667. } else {
  1668. rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
  1669. 0, PASS_ONE_BIT,
  1670. &bit_chk, 0);
  1671. bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
  1672. (read_group - (write_group *
  1673. RW_MGR_MEM_IF_READ_DQS_WIDTH /
  1674. RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
  1675. stop = (bit_chk == 0);
  1676. }
  1677. sticky_bit_chk = sticky_bit_chk | bit_chk;
  1678. stop = stop && (sticky_bit_chk == param->read_correct_mask);
  1679. debug_cond(DLEVEL == 2, "%s:%d vfifo_center(right): dtap=%u => %u == \
  1680. %u && %u", __func__, __LINE__, d,
  1681. sticky_bit_chk, param->read_correct_mask, stop);
  1682. if (stop == 1) {
  1683. break;
  1684. } else {
  1685. for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
  1686. if (bit_chk & 1) {
  1687. /* Remember a passing test as
  1688. the right_edge */
  1689. right_edge[i] = d;
  1690. } else {
  1691. if (d != 0) {
  1692. /* If a right edge has not been
  1693. seen yet, then a future passing
  1694. test will mark this edge as the
  1695. left edge */
  1696. if (right_edge[i] ==
  1697. IO_IO_IN_DELAY_MAX + 1) {
  1698. left_edge[i] = -(d + 1);
  1699. }
  1700. } else {
  1701. /* d = 0 failed, but it passed
  1702. when testing the left edge,
  1703. so it must be marginal,
  1704. set it to -1 */
  1705. if (right_edge[i] ==
  1706. IO_IO_IN_DELAY_MAX + 1 &&
  1707. left_edge[i] !=
  1708. IO_IO_IN_DELAY_MAX
  1709. + 1) {
  1710. right_edge[i] = -1;
  1711. }
  1712. /* If a right edge has not been
  1713. seen yet, then a future passing
  1714. test will mark this edge as the
  1715. left edge */
  1716. else if (right_edge[i] ==
  1717. IO_IO_IN_DELAY_MAX +
  1718. 1) {
  1719. left_edge[i] = -(d + 1);
  1720. }
  1721. }
  1722. }
  1723. debug_cond(DLEVEL == 2, "%s:%d vfifo_center[r,\
  1724. d=%u]: ", __func__, __LINE__, d);
  1725. debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d ",
  1726. (int)(bit_chk & 1), i, left_edge[i]);
  1727. debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
  1728. right_edge[i]);
  1729. bit_chk = bit_chk >> 1;
  1730. }
  1731. }
  1732. }
  1733. /* Check that all bits have a window */
  1734. for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
  1735. debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
  1736. %d right_edge[%u]: %d", __func__, __LINE__,
  1737. i, left_edge[i], i, right_edge[i]);
  1738. if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) || (right_edge[i]
  1739. == IO_IO_IN_DELAY_MAX + 1)) {
  1740. /*
  1741. * Restore delay chain settings before letting the loop
  1742. * in rw_mgr_mem_calibrate_vfifo to retry different
  1743. * dqs/ck relationships.
  1744. */
  1745. scc_mgr_set_dqs_bus_in_delay(read_group, start_dqs);
  1746. if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
  1747. scc_mgr_set_dqs_en_delay(read_group,
  1748. start_dqs_en);
  1749. }
  1750. scc_mgr_load_dqs(read_group);
  1751. writel(0, &sdr_scc_mgr->update);
  1752. debug_cond(DLEVEL == 1, "%s:%d vfifo_center: failed to \
  1753. find edge [%u]: %d %d", __func__, __LINE__,
  1754. i, left_edge[i], right_edge[i]);
  1755. if (use_read_test) {
  1756. set_failing_group_stage(read_group *
  1757. RW_MGR_MEM_DQ_PER_READ_DQS + i,
  1758. CAL_STAGE_VFIFO,
  1759. CAL_SUBSTAGE_VFIFO_CENTER);
  1760. } else {
  1761. set_failing_group_stage(read_group *
  1762. RW_MGR_MEM_DQ_PER_READ_DQS + i,
  1763. CAL_STAGE_VFIFO_AFTER_WRITES,
  1764. CAL_SUBSTAGE_VFIFO_CENTER);
  1765. }
  1766. return 0;
  1767. }
  1768. }
  1769. /* Find middle of window for each DQ bit */
  1770. mid_min = left_edge[0] - right_edge[0];
  1771. min_index = 0;
  1772. for (i = 1; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
  1773. mid = left_edge[i] - right_edge[i];
  1774. if (mid < mid_min) {
  1775. mid_min = mid;
  1776. min_index = i;
  1777. }
  1778. }
  1779. /*
  1780. * -mid_min/2 represents the amount that we need to move DQS.
  1781. * If mid_min is odd and positive we'll need to add one to
  1782. * make sure the rounding in further calculations is correct
  1783. * (always bias to the right), so just add 1 for all positive values.
  1784. */
  1785. if (mid_min > 0)
  1786. mid_min++;
  1787. mid_min = mid_min / 2;
  1788. debug_cond(DLEVEL == 1, "%s:%d vfifo_center: mid_min=%d (index=%u)\n",
  1789. __func__, __LINE__, mid_min, min_index);
  1790. /* Determine the amount we can change DQS (which is -mid_min) */
  1791. orig_mid_min = mid_min;
  1792. new_dqs = start_dqs - mid_min;
  1793. if (new_dqs > IO_DQS_IN_DELAY_MAX)
  1794. new_dqs = IO_DQS_IN_DELAY_MAX;
  1795. else if (new_dqs < 0)
  1796. new_dqs = 0;
  1797. mid_min = start_dqs - new_dqs;
  1798. debug_cond(DLEVEL == 1, "vfifo_center: new mid_min=%d new_dqs=%d\n",
  1799. mid_min, new_dqs);
  1800. if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
  1801. if (start_dqs_en - mid_min > IO_DQS_EN_DELAY_MAX)
  1802. mid_min += start_dqs_en - mid_min - IO_DQS_EN_DELAY_MAX;
  1803. else if (start_dqs_en - mid_min < 0)
  1804. mid_min += start_dqs_en - mid_min;
  1805. }
  1806. new_dqs = start_dqs - mid_min;
  1807. debug_cond(DLEVEL == 1, "vfifo_center: start_dqs=%d start_dqs_en=%d \
  1808. new_dqs=%d mid_min=%d\n", start_dqs,
  1809. IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS ? start_dqs_en : -1,
  1810. new_dqs, mid_min);
  1811. /* Initialize data for export structures */
  1812. dqs_margin = IO_IO_IN_DELAY_MAX + 1;
  1813. dq_margin = IO_IO_IN_DELAY_MAX + 1;
  1814. /* add delay to bring centre of all DQ windows to the same "level" */
  1815. for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
  1816. /* Use values before divide by 2 to reduce round off error */
  1817. shift_dq = (left_edge[i] - right_edge[i] -
  1818. (left_edge[min_index] - right_edge[min_index]))/2 +
  1819. (orig_mid_min - mid_min);
  1820. debug_cond(DLEVEL == 2, "vfifo_center: before: \
  1821. shift_dq[%u]=%d\n", i, shift_dq);
  1822. addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_IN_DELAY_OFFSET;
  1823. temp_dq_in_delay1 = readl(addr + (p << 2));
  1824. temp_dq_in_delay2 = readl(addr + (i << 2));
  1825. if (shift_dq + (int32_t)temp_dq_in_delay1 >
  1826. (int32_t)IO_IO_IN_DELAY_MAX) {
  1827. shift_dq = (int32_t)IO_IO_IN_DELAY_MAX - temp_dq_in_delay2;
  1828. } else if (shift_dq + (int32_t)temp_dq_in_delay1 < 0) {
  1829. shift_dq = -(int32_t)temp_dq_in_delay1;
  1830. }
  1831. debug_cond(DLEVEL == 2, "vfifo_center: after: \
  1832. shift_dq[%u]=%d\n", i, shift_dq);
  1833. final_dq[i] = temp_dq_in_delay1 + shift_dq;
  1834. scc_mgr_set_dq_in_delay(p, final_dq[i]);
  1835. scc_mgr_load_dq(p);
  1836. debug_cond(DLEVEL == 2, "vfifo_center: margin[%u]=[%d,%d]\n", i,
  1837. left_edge[i] - shift_dq + (-mid_min),
  1838. right_edge[i] + shift_dq - (-mid_min));
  1839. /* To determine values for export structures */
  1840. if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
  1841. dq_margin = left_edge[i] - shift_dq + (-mid_min);
  1842. if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
  1843. dqs_margin = right_edge[i] + shift_dq - (-mid_min);
  1844. }
  1845. final_dqs = new_dqs;
  1846. if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
  1847. final_dqs_en = start_dqs_en - mid_min;
  1848. /* Move DQS-en */
  1849. if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
  1850. scc_mgr_set_dqs_en_delay(read_group, final_dqs_en);
  1851. scc_mgr_load_dqs(read_group);
  1852. }
  1853. /* Move DQS */
  1854. scc_mgr_set_dqs_bus_in_delay(read_group, final_dqs);
  1855. scc_mgr_load_dqs(read_group);
  1856. debug_cond(DLEVEL == 2, "%s:%d vfifo_center: dq_margin=%d \
  1857. dqs_margin=%d", __func__, __LINE__,
  1858. dq_margin, dqs_margin);
  1859. /*
  1860. * Do not remove this line as it makes sure all of our decisions
  1861. * have been applied. Apply the update bit.
  1862. */
  1863. writel(0, &sdr_scc_mgr->update);
  1864. return (dq_margin >= 0) && (dqs_margin >= 0);
  1865. }
  1866. /*
  1867. * calibrate the read valid prediction FIFO.
  1868. *
  1869. * - read valid prediction will consist of finding a good DQS enable phase,
  1870. * DQS enable delay, DQS input phase, and DQS input delay.
  1871. * - we also do a per-bit deskew on the DQ lines.
  1872. */
  1873. static uint32_t rw_mgr_mem_calibrate_vfifo(uint32_t read_group,
  1874. uint32_t test_bgn)
  1875. {
  1876. uint32_t p, d, rank_bgn, sr;
  1877. uint32_t dtaps_per_ptap;
  1878. uint32_t tmp_delay;
  1879. uint32_t bit_chk;
  1880. uint32_t grp_calibrated;
  1881. uint32_t write_group, write_test_bgn;
  1882. uint32_t failed_substage;
  1883. debug("%s:%d: %u %u\n", __func__, __LINE__, read_group, test_bgn);
  1884. /* update info for sims */
  1885. reg_file_set_stage(CAL_STAGE_VFIFO);
  1886. write_group = read_group;
  1887. write_test_bgn = test_bgn;
  1888. /* USER Determine number of delay taps for each phase tap */
  1889. dtaps_per_ptap = 0;
  1890. tmp_delay = 0;
  1891. while (tmp_delay < IO_DELAY_PER_OPA_TAP) {
  1892. dtaps_per_ptap++;
  1893. tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
  1894. }
  1895. dtaps_per_ptap--;
  1896. tmp_delay = 0;
  1897. /* update info for sims */
  1898. reg_file_set_group(read_group);
  1899. grp_calibrated = 0;
  1900. reg_file_set_sub_stage(CAL_SUBSTAGE_GUARANTEED_READ);
  1901. failed_substage = CAL_SUBSTAGE_GUARANTEED_READ;
  1902. for (d = 0; d <= dtaps_per_ptap && grp_calibrated == 0; d += 2) {
  1903. /*
  1904. * In RLDRAMX we may be messing the delay of pins in
  1905. * the same write group but outside of the current read
  1906. * the group, but that's ok because we haven't
  1907. * calibrated output side yet.
  1908. */
  1909. if (d > 0) {
  1910. scc_mgr_apply_group_all_out_delay_add_all_ranks(
  1911. write_group, d);
  1912. }
  1913. for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX && grp_calibrated == 0;
  1914. p++) {
  1915. /* set a particular dqdqs phase */
  1916. scc_mgr_set_dqdqs_output_phase_all_ranks(read_group, p);
  1917. debug_cond(DLEVEL == 1, "%s:%d calibrate_vfifo: g=%u \
  1918. p=%u d=%u\n", __func__, __LINE__,
  1919. read_group, p, d);
  1920. /*
  1921. * Load up the patterns used by read calibration
  1922. * using current DQDQS phase.
  1923. */
  1924. rw_mgr_mem_calibrate_read_load_patterns(0, 1);
  1925. if (!(gbl->phy_debug_mode_flags &
  1926. PHY_DEBUG_DISABLE_GUARANTEED_READ)) {
  1927. if (!rw_mgr_mem_calibrate_read_test_patterns_all_ranks
  1928. (read_group, 1, &bit_chk)) {
  1929. debug_cond(DLEVEL == 1, "%s:%d Guaranteed read test failed:",
  1930. __func__, __LINE__);
  1931. debug_cond(DLEVEL == 1, " g=%u p=%u d=%u\n",
  1932. read_group, p, d);
  1933. break;
  1934. }
  1935. }
  1936. /* case:56390 */
  1937. grp_calibrated = 1;
  1938. if (rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay
  1939. (write_group, read_group, test_bgn)) {
  1940. /*
  1941. * USER Read per-bit deskew can be done on a
  1942. * per shadow register basis.
  1943. */
  1944. for (rank_bgn = 0, sr = 0;
  1945. rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
  1946. rank_bgn += NUM_RANKS_PER_SHADOW_REG,
  1947. ++sr) {
  1948. /*
  1949. * Determine if this set of ranks
  1950. * should be skipped entirely.
  1951. */
  1952. if (!param->skip_shadow_regs[sr]) {
  1953. /*
  1954. * If doing read after write
  1955. * calibration, do not update
  1956. * FOM, now - do it then.
  1957. */
  1958. if (!rw_mgr_mem_calibrate_vfifo_center
  1959. (rank_bgn, write_group,
  1960. read_group, test_bgn, 1, 0)) {
  1961. grp_calibrated = 0;
  1962. failed_substage =
  1963. CAL_SUBSTAGE_VFIFO_CENTER;
  1964. }
  1965. }
  1966. }
  1967. } else {
  1968. grp_calibrated = 0;
  1969. failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE;
  1970. }
  1971. }
  1972. }
  1973. if (grp_calibrated == 0) {
  1974. set_failing_group_stage(write_group, CAL_STAGE_VFIFO,
  1975. failed_substage);
  1976. return 0;
  1977. }
  1978. /*
  1979. * Reset the delay chains back to zero if they have moved > 1
  1980. * (check for > 1 because loop will increase d even when pass in
  1981. * first case).
  1982. */
  1983. if (d > 2)
  1984. scc_mgr_zero_group(write_group, 1);
  1985. return 1;
  1986. }
  1987. /* VFIFO Calibration -- Read Deskew Calibration after write deskew */
  1988. static uint32_t rw_mgr_mem_calibrate_vfifo_end(uint32_t read_group,
  1989. uint32_t test_bgn)
  1990. {
  1991. uint32_t rank_bgn, sr;
  1992. uint32_t grp_calibrated;
  1993. uint32_t write_group;
  1994. debug("%s:%d %u %u", __func__, __LINE__, read_group, test_bgn);
  1995. /* update info for sims */
  1996. reg_file_set_stage(CAL_STAGE_VFIFO_AFTER_WRITES);
  1997. reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
  1998. write_group = read_group;
  1999. /* update info for sims */
  2000. reg_file_set_group(read_group);
  2001. grp_calibrated = 1;
  2002. /* Read per-bit deskew can be done on a per shadow register basis */
  2003. for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
  2004. rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) {
  2005. /* Determine if this set of ranks should be skipped entirely */
  2006. if (!param->skip_shadow_regs[sr]) {
  2007. /* This is the last calibration round, update FOM here */
  2008. if (!rw_mgr_mem_calibrate_vfifo_center(rank_bgn,
  2009. write_group,
  2010. read_group,
  2011. test_bgn, 0,
  2012. 1)) {
  2013. grp_calibrated = 0;
  2014. }
  2015. }
  2016. }
  2017. if (grp_calibrated == 0) {
  2018. set_failing_group_stage(write_group,
  2019. CAL_STAGE_VFIFO_AFTER_WRITES,
  2020. CAL_SUBSTAGE_VFIFO_CENTER);
  2021. return 0;
  2022. }
  2023. return 1;
  2024. }
  2025. /* Calibrate LFIFO to find smallest read latency */
  2026. static uint32_t rw_mgr_mem_calibrate_lfifo(void)
  2027. {
  2028. uint32_t found_one;
  2029. uint32_t bit_chk;
  2030. debug("%s:%d\n", __func__, __LINE__);
  2031. /* update info for sims */
  2032. reg_file_set_stage(CAL_STAGE_LFIFO);
  2033. reg_file_set_sub_stage(CAL_SUBSTAGE_READ_LATENCY);
  2034. /* Load up the patterns used by read calibration for all ranks */
  2035. rw_mgr_mem_calibrate_read_load_patterns(0, 1);
  2036. found_one = 0;
  2037. do {
  2038. writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
  2039. debug_cond(DLEVEL == 2, "%s:%d lfifo: read_lat=%u",
  2040. __func__, __LINE__, gbl->curr_read_lat);
  2041. if (!rw_mgr_mem_calibrate_read_test_all_ranks(0,
  2042. NUM_READ_TESTS,
  2043. PASS_ALL_BITS,
  2044. &bit_chk, 1)) {
  2045. break;
  2046. }
  2047. found_one = 1;
  2048. /* reduce read latency and see if things are working */
  2049. /* correctly */
  2050. gbl->curr_read_lat--;
  2051. } while (gbl->curr_read_lat > 0);
  2052. /* reset the fifos to get pointers to known state */
  2053. writel(0, &phy_mgr_cmd->fifo_reset);
  2054. if (found_one) {
  2055. /* add a fudge factor to the read latency that was determined */
  2056. gbl->curr_read_lat += 2;
  2057. writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
  2058. debug_cond(DLEVEL == 2, "%s:%d lfifo: success: using \
  2059. read_lat=%u\n", __func__, __LINE__,
  2060. gbl->curr_read_lat);
  2061. return 1;
  2062. } else {
  2063. set_failing_group_stage(0xff, CAL_STAGE_LFIFO,
  2064. CAL_SUBSTAGE_READ_LATENCY);
  2065. debug_cond(DLEVEL == 2, "%s:%d lfifo: failed at initial \
  2066. read_lat=%u\n", __func__, __LINE__,
  2067. gbl->curr_read_lat);
  2068. return 0;
  2069. }
  2070. }
  2071. /*
  2072. * issue write test command.
  2073. * two variants are provided. one that just tests a write pattern and
  2074. * another that tests datamask functionality.
  2075. */
  2076. static void rw_mgr_mem_calibrate_write_test_issue(uint32_t group,
  2077. uint32_t test_dm)
  2078. {
  2079. uint32_t mcc_instruction;
  2080. uint32_t quick_write_mode = (((STATIC_CALIB_STEPS) & CALIB_SKIP_WRITES) &&
  2081. ENABLE_SUPER_QUICK_CALIBRATION);
  2082. uint32_t rw_wl_nop_cycles;
  2083. uint32_t addr;
  2084. /*
  2085. * Set counter and jump addresses for the right
  2086. * number of NOP cycles.
  2087. * The number of supported NOP cycles can range from -1 to infinity
  2088. * Three different cases are handled:
  2089. *
  2090. * 1. For a number of NOP cycles greater than 0, the RW Mgr looping
  2091. * mechanism will be used to insert the right number of NOPs
  2092. *
  2093. * 2. For a number of NOP cycles equals to 0, the micro-instruction
  2094. * issuing the write command will jump straight to the
  2095. * micro-instruction that turns on DQS (for DDRx), or outputs write
  2096. * data (for RLD), skipping
  2097. * the NOP micro-instruction all together
  2098. *
  2099. * 3. A number of NOP cycles equal to -1 indicates that DQS must be
  2100. * turned on in the same micro-instruction that issues the write
  2101. * command. Then we need
  2102. * to directly jump to the micro-instruction that sends out the data
  2103. *
  2104. * NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters
  2105. * (2 and 3). One jump-counter (0) is used to perform multiple
  2106. * write-read operations.
  2107. * one counter left to issue this command in "multiple-group" mode
  2108. */
  2109. rw_wl_nop_cycles = gbl->rw_wl_nop_cycles;
  2110. if (rw_wl_nop_cycles == -1) {
  2111. /*
  2112. * CNTR 2 - We want to execute the special write operation that
  2113. * turns on DQS right away and then skip directly to the
  2114. * instruction that sends out the data. We set the counter to a
  2115. * large number so that the jump is always taken.
  2116. */
  2117. writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
  2118. /* CNTR 3 - Not used */
  2119. if (test_dm) {
  2120. mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0_WL_1;
  2121. writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DATA,
  2122. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  2123. writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP,
  2124. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  2125. } else {
  2126. mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0_WL_1;
  2127. writel(RW_MGR_LFSR_WR_RD_BANK_0_DATA,
  2128. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  2129. writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP,
  2130. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  2131. }
  2132. } else if (rw_wl_nop_cycles == 0) {
  2133. /*
  2134. * CNTR 2 - We want to skip the NOP operation and go straight
  2135. * to the DQS enable instruction. We set the counter to a large
  2136. * number so that the jump is always taken.
  2137. */
  2138. writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
  2139. /* CNTR 3 - Not used */
  2140. if (test_dm) {
  2141. mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
  2142. writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DQS,
  2143. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  2144. } else {
  2145. mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
  2146. writel(RW_MGR_LFSR_WR_RD_BANK_0_DQS,
  2147. &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  2148. }
  2149. } else {
  2150. /*
  2151. * CNTR 2 - In this case we want to execute the next instruction
  2152. * and NOT take the jump. So we set the counter to 0. The jump
  2153. * address doesn't count.
  2154. */
  2155. writel(0x0, &sdr_rw_load_mgr_regs->load_cntr2);
  2156. writel(0x0, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
  2157. /*
  2158. * CNTR 3 - Set the nop counter to the number of cycles we
  2159. * need to loop for, minus 1.
  2160. */
  2161. writel(rw_wl_nop_cycles - 1, &sdr_rw_load_mgr_regs->load_cntr3);
  2162. if (test_dm) {
  2163. mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
  2164. writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP,
  2165. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  2166. } else {
  2167. mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
  2168. writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP,
  2169. &sdr_rw_load_jump_mgr_regs->load_jump_add3);
  2170. }
  2171. }
  2172. writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  2173. RW_MGR_RESET_READ_DATAPATH_OFFSET);
  2174. if (quick_write_mode)
  2175. writel(0x08, &sdr_rw_load_mgr_regs->load_cntr0);
  2176. else
  2177. writel(0x40, &sdr_rw_load_mgr_regs->load_cntr0);
  2178. writel(mcc_instruction, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  2179. /*
  2180. * CNTR 1 - This is used to ensure enough time elapses
  2181. * for read data to come back.
  2182. */
  2183. writel(0x30, &sdr_rw_load_mgr_regs->load_cntr1);
  2184. if (test_dm) {
  2185. writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_WAIT,
  2186. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  2187. } else {
  2188. writel(RW_MGR_LFSR_WR_RD_BANK_0_WAIT,
  2189. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  2190. }
  2191. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
  2192. writel(mcc_instruction, addr + (group << 2));
  2193. }
  2194. /* Test writes, can check for a single bit pass or multiple bit pass */
  2195. static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
  2196. uint32_t write_group, uint32_t use_dm, uint32_t all_correct,
  2197. uint32_t *bit_chk, uint32_t all_ranks)
  2198. {
  2199. uint32_t r;
  2200. uint32_t correct_mask_vg;
  2201. uint32_t tmp_bit_chk;
  2202. uint32_t vg;
  2203. uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
  2204. (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
  2205. uint32_t addr_rw_mgr;
  2206. uint32_t base_rw_mgr;
  2207. *bit_chk = param->write_correct_mask;
  2208. correct_mask_vg = param->write_correct_mask_vg;
  2209. for (r = rank_bgn; r < rank_end; r++) {
  2210. if (param->skip_ranks[r]) {
  2211. /* request to skip the rank */
  2212. continue;
  2213. }
  2214. /* set rank */
  2215. set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
  2216. tmp_bit_chk = 0;
  2217. addr_rw_mgr = SDR_PHYGRP_RWMGRGRP_ADDRESS;
  2218. for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS-1; ; vg--) {
  2219. /* reset the fifos to get pointers to known state */
  2220. writel(0, &phy_mgr_cmd->fifo_reset);
  2221. tmp_bit_chk = tmp_bit_chk <<
  2222. (RW_MGR_MEM_DQ_PER_WRITE_DQS /
  2223. RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
  2224. rw_mgr_mem_calibrate_write_test_issue(write_group *
  2225. RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS+vg,
  2226. use_dm);
  2227. base_rw_mgr = readl(addr_rw_mgr);
  2228. tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
  2229. if (vg == 0)
  2230. break;
  2231. }
  2232. *bit_chk &= tmp_bit_chk;
  2233. }
  2234. if (all_correct) {
  2235. set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
  2236. debug_cond(DLEVEL == 2, "write_test(%u,%u,ALL) : %u == \
  2237. %u => %lu", write_group, use_dm,
  2238. *bit_chk, param->write_correct_mask,
  2239. (long unsigned int)(*bit_chk ==
  2240. param->write_correct_mask));
  2241. return *bit_chk == param->write_correct_mask;
  2242. } else {
  2243. set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
  2244. debug_cond(DLEVEL == 2, "write_test(%u,%u,ONE) : %u != ",
  2245. write_group, use_dm, *bit_chk);
  2246. debug_cond(DLEVEL == 2, "%lu" " => %lu", (long unsigned int)0,
  2247. (long unsigned int)(*bit_chk != 0));
  2248. return *bit_chk != 0x00;
  2249. }
  2250. }
  2251. /*
  2252. * center all windows. do per-bit-deskew to possibly increase size of
  2253. * certain windows.
  2254. */
  2255. static uint32_t rw_mgr_mem_calibrate_writes_center(uint32_t rank_bgn,
  2256. uint32_t write_group, uint32_t test_bgn)
  2257. {
  2258. uint32_t i, p, min_index;
  2259. int32_t d;
  2260. /*
  2261. * Store these as signed since there are comparisons with
  2262. * signed numbers.
  2263. */
  2264. uint32_t bit_chk;
  2265. uint32_t sticky_bit_chk;
  2266. int32_t left_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
  2267. int32_t right_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
  2268. int32_t mid;
  2269. int32_t mid_min, orig_mid_min;
  2270. int32_t new_dqs, start_dqs, shift_dq;
  2271. int32_t dq_margin, dqs_margin, dm_margin;
  2272. uint32_t stop;
  2273. uint32_t temp_dq_out1_delay;
  2274. uint32_t addr;
  2275. debug("%s:%d %u %u", __func__, __LINE__, write_group, test_bgn);
  2276. dm_margin = 0;
  2277. addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
  2278. start_dqs = readl(addr +
  2279. (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2));
  2280. /* per-bit deskew */
  2281. /*
  2282. * set the left and right edge of each bit to an illegal value
  2283. * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value.
  2284. */
  2285. sticky_bit_chk = 0;
  2286. for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
  2287. left_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
  2288. right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
  2289. }
  2290. /* Search for the left edge of the window for each bit */
  2291. for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++) {
  2292. scc_mgr_apply_group_dq_out1_delay(write_group, d);
  2293. writel(0, &sdr_scc_mgr->update);
  2294. /*
  2295. * Stop searching when the read test doesn't pass AND when
  2296. * we've seen a passing read on every bit.
  2297. */
  2298. stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
  2299. 0, PASS_ONE_BIT, &bit_chk, 0);
  2300. sticky_bit_chk = sticky_bit_chk | bit_chk;
  2301. stop = stop && (sticky_bit_chk == param->write_correct_mask);
  2302. debug_cond(DLEVEL == 2, "write_center(left): dtap=%d => %u \
  2303. == %u && %u [bit_chk= %u ]\n",
  2304. d, sticky_bit_chk, param->write_correct_mask,
  2305. stop, bit_chk);
  2306. if (stop == 1) {
  2307. break;
  2308. } else {
  2309. for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
  2310. if (bit_chk & 1) {
  2311. /*
  2312. * Remember a passing test as the
  2313. * left_edge.
  2314. */
  2315. left_edge[i] = d;
  2316. } else {
  2317. /*
  2318. * If a left edge has not been seen
  2319. * yet, then a future passing test will
  2320. * mark this edge as the right edge.
  2321. */
  2322. if (left_edge[i] ==
  2323. IO_IO_OUT1_DELAY_MAX + 1) {
  2324. right_edge[i] = -(d + 1);
  2325. }
  2326. }
  2327. debug_cond(DLEVEL == 2, "write_center[l,d=%d):", d);
  2328. debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
  2329. (int)(bit_chk & 1), i, left_edge[i]);
  2330. debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
  2331. right_edge[i]);
  2332. bit_chk = bit_chk >> 1;
  2333. }
  2334. }
  2335. }
  2336. /* Reset DQ delay chains to 0 */
  2337. scc_mgr_apply_group_dq_out1_delay(0);
  2338. sticky_bit_chk = 0;
  2339. for (i = RW_MGR_MEM_DQ_PER_WRITE_DQS - 1;; i--) {
  2340. debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
  2341. %d right_edge[%u]: %d\n", __func__, __LINE__,
  2342. i, left_edge[i], i, right_edge[i]);
  2343. /*
  2344. * Check for cases where we haven't found the left edge,
  2345. * which makes our assignment of the the right edge invalid.
  2346. * Reset it to the illegal value.
  2347. */
  2348. if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) &&
  2349. (right_edge[i] != IO_IO_OUT1_DELAY_MAX + 1)) {
  2350. right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
  2351. debug_cond(DLEVEL == 2, "%s:%d write_center: reset \
  2352. right_edge[%u]: %d\n", __func__, __LINE__,
  2353. i, right_edge[i]);
  2354. }
  2355. /*
  2356. * Reset sticky bit (except for bits where we have
  2357. * seen the left edge).
  2358. */
  2359. sticky_bit_chk = sticky_bit_chk << 1;
  2360. if ((left_edge[i] != IO_IO_OUT1_DELAY_MAX + 1))
  2361. sticky_bit_chk = sticky_bit_chk | 1;
  2362. if (i == 0)
  2363. break;
  2364. }
  2365. /* Search for the right edge of the window for each bit */
  2366. for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - start_dqs; d++) {
  2367. scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
  2368. d + start_dqs);
  2369. writel(0, &sdr_scc_mgr->update);
  2370. /*
  2371. * Stop searching when the read test doesn't pass AND when
  2372. * we've seen a passing read on every bit.
  2373. */
  2374. stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
  2375. 0, PASS_ONE_BIT, &bit_chk, 0);
  2376. sticky_bit_chk = sticky_bit_chk | bit_chk;
  2377. stop = stop && (sticky_bit_chk == param->write_correct_mask);
  2378. debug_cond(DLEVEL == 2, "write_center (right): dtap=%u => %u == \
  2379. %u && %u\n", d, sticky_bit_chk,
  2380. param->write_correct_mask, stop);
  2381. if (stop == 1) {
  2382. if (d == 0) {
  2383. for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS;
  2384. i++) {
  2385. /* d = 0 failed, but it passed when
  2386. testing the left edge, so it must be
  2387. marginal, set it to -1 */
  2388. if (right_edge[i] ==
  2389. IO_IO_OUT1_DELAY_MAX + 1 &&
  2390. left_edge[i] !=
  2391. IO_IO_OUT1_DELAY_MAX + 1) {
  2392. right_edge[i] = -1;
  2393. }
  2394. }
  2395. }
  2396. break;
  2397. } else {
  2398. for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
  2399. if (bit_chk & 1) {
  2400. /*
  2401. * Remember a passing test as
  2402. * the right_edge.
  2403. */
  2404. right_edge[i] = d;
  2405. } else {
  2406. if (d != 0) {
  2407. /*
  2408. * If a right edge has not
  2409. * been seen yet, then a future
  2410. * passing test will mark this
  2411. * edge as the left edge.
  2412. */
  2413. if (right_edge[i] ==
  2414. IO_IO_OUT1_DELAY_MAX + 1)
  2415. left_edge[i] = -(d + 1);
  2416. } else {
  2417. /*
  2418. * d = 0 failed, but it passed
  2419. * when testing the left edge,
  2420. * so it must be marginal, set
  2421. * it to -1.
  2422. */
  2423. if (right_edge[i] ==
  2424. IO_IO_OUT1_DELAY_MAX + 1 &&
  2425. left_edge[i] !=
  2426. IO_IO_OUT1_DELAY_MAX + 1)
  2427. right_edge[i] = -1;
  2428. /*
  2429. * If a right edge has not been
  2430. * seen yet, then a future
  2431. * passing test will mark this
  2432. * edge as the left edge.
  2433. */
  2434. else if (right_edge[i] ==
  2435. IO_IO_OUT1_DELAY_MAX +
  2436. 1)
  2437. left_edge[i] = -(d + 1);
  2438. }
  2439. }
  2440. debug_cond(DLEVEL == 2, "write_center[r,d=%d):", d);
  2441. debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
  2442. (int)(bit_chk & 1), i, left_edge[i]);
  2443. debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
  2444. right_edge[i]);
  2445. bit_chk = bit_chk >> 1;
  2446. }
  2447. }
  2448. }
  2449. /* Check that all bits have a window */
  2450. for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
  2451. debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
  2452. %d right_edge[%u]: %d", __func__, __LINE__,
  2453. i, left_edge[i], i, right_edge[i]);
  2454. if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) ||
  2455. (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1)) {
  2456. set_failing_group_stage(test_bgn + i,
  2457. CAL_STAGE_WRITES,
  2458. CAL_SUBSTAGE_WRITES_CENTER);
  2459. return 0;
  2460. }
  2461. }
  2462. /* Find middle of window for each DQ bit */
  2463. mid_min = left_edge[0] - right_edge[0];
  2464. min_index = 0;
  2465. for (i = 1; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
  2466. mid = left_edge[i] - right_edge[i];
  2467. if (mid < mid_min) {
  2468. mid_min = mid;
  2469. min_index = i;
  2470. }
  2471. }
  2472. /*
  2473. * -mid_min/2 represents the amount that we need to move DQS.
  2474. * If mid_min is odd and positive we'll need to add one to
  2475. * make sure the rounding in further calculations is correct
  2476. * (always bias to the right), so just add 1 for all positive values.
  2477. */
  2478. if (mid_min > 0)
  2479. mid_min++;
  2480. mid_min = mid_min / 2;
  2481. debug_cond(DLEVEL == 1, "%s:%d write_center: mid_min=%d\n", __func__,
  2482. __LINE__, mid_min);
  2483. /* Determine the amount we can change DQS (which is -mid_min) */
  2484. orig_mid_min = mid_min;
  2485. new_dqs = start_dqs;
  2486. mid_min = 0;
  2487. debug_cond(DLEVEL == 1, "%s:%d write_center: start_dqs=%d new_dqs=%d \
  2488. mid_min=%d\n", __func__, __LINE__, start_dqs, new_dqs, mid_min);
  2489. /* Initialize data for export structures */
  2490. dqs_margin = IO_IO_OUT1_DELAY_MAX + 1;
  2491. dq_margin = IO_IO_OUT1_DELAY_MAX + 1;
  2492. /* add delay to bring centre of all DQ windows to the same "level" */
  2493. for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
  2494. /* Use values before divide by 2 to reduce round off error */
  2495. shift_dq = (left_edge[i] - right_edge[i] -
  2496. (left_edge[min_index] - right_edge[min_index]))/2 +
  2497. (orig_mid_min - mid_min);
  2498. debug_cond(DLEVEL == 2, "%s:%d write_center: before: shift_dq \
  2499. [%u]=%d\n", __func__, __LINE__, i, shift_dq);
  2500. addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
  2501. temp_dq_out1_delay = readl(addr + (i << 2));
  2502. if (shift_dq + (int32_t)temp_dq_out1_delay >
  2503. (int32_t)IO_IO_OUT1_DELAY_MAX) {
  2504. shift_dq = (int32_t)IO_IO_OUT1_DELAY_MAX - temp_dq_out1_delay;
  2505. } else if (shift_dq + (int32_t)temp_dq_out1_delay < 0) {
  2506. shift_dq = -(int32_t)temp_dq_out1_delay;
  2507. }
  2508. debug_cond(DLEVEL == 2, "write_center: after: shift_dq[%u]=%d\n",
  2509. i, shift_dq);
  2510. scc_mgr_set_dq_out1_delay(i, temp_dq_out1_delay + shift_dq);
  2511. scc_mgr_load_dq(i);
  2512. debug_cond(DLEVEL == 2, "write_center: margin[%u]=[%d,%d]\n", i,
  2513. left_edge[i] - shift_dq + (-mid_min),
  2514. right_edge[i] + shift_dq - (-mid_min));
  2515. /* To determine values for export structures */
  2516. if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
  2517. dq_margin = left_edge[i] - shift_dq + (-mid_min);
  2518. if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
  2519. dqs_margin = right_edge[i] + shift_dq - (-mid_min);
  2520. }
  2521. /* Move DQS */
  2522. scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
  2523. writel(0, &sdr_scc_mgr->update);
  2524. /* Centre DM */
  2525. debug_cond(DLEVEL == 2, "%s:%d write_center: DM\n", __func__, __LINE__);
  2526. /*
  2527. * set the left and right edge of each bit to an illegal value,
  2528. * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value,
  2529. */
  2530. left_edge[0] = IO_IO_OUT1_DELAY_MAX + 1;
  2531. right_edge[0] = IO_IO_OUT1_DELAY_MAX + 1;
  2532. int32_t bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
  2533. int32_t end_curr = IO_IO_OUT1_DELAY_MAX + 1;
  2534. int32_t bgn_best = IO_IO_OUT1_DELAY_MAX + 1;
  2535. int32_t end_best = IO_IO_OUT1_DELAY_MAX + 1;
  2536. int32_t win_best = 0;
  2537. /* Search for the/part of the window with DM shift */
  2538. for (d = IO_IO_OUT1_DELAY_MAX; d >= 0; d -= DELTA_D) {
  2539. scc_mgr_apply_group_dm_out1_delay(d);
  2540. writel(0, &sdr_scc_mgr->update);
  2541. if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
  2542. PASS_ALL_BITS, &bit_chk,
  2543. 0)) {
  2544. /* USE Set current end of the window */
  2545. end_curr = -d;
  2546. /*
  2547. * If a starting edge of our window has not been seen
  2548. * this is our current start of the DM window.
  2549. */
  2550. if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
  2551. bgn_curr = -d;
  2552. /*
  2553. * If current window is bigger than best seen.
  2554. * Set best seen to be current window.
  2555. */
  2556. if ((end_curr-bgn_curr+1) > win_best) {
  2557. win_best = end_curr-bgn_curr+1;
  2558. bgn_best = bgn_curr;
  2559. end_best = end_curr;
  2560. }
  2561. } else {
  2562. /* We just saw a failing test. Reset temp edge */
  2563. bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
  2564. end_curr = IO_IO_OUT1_DELAY_MAX + 1;
  2565. }
  2566. }
  2567. /* Reset DM delay chains to 0 */
  2568. scc_mgr_apply_group_dm_out1_delay(0);
  2569. /*
  2570. * Check to see if the current window nudges up aganist 0 delay.
  2571. * If so we need to continue the search by shifting DQS otherwise DQS
  2572. * search begins as a new search. */
  2573. if (end_curr != 0) {
  2574. bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
  2575. end_curr = IO_IO_OUT1_DELAY_MAX + 1;
  2576. }
  2577. /* Search for the/part of the window with DQS shifts */
  2578. for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - new_dqs; d += DELTA_D) {
  2579. /*
  2580. * Note: This only shifts DQS, so are we limiting ourselve to
  2581. * width of DQ unnecessarily.
  2582. */
  2583. scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
  2584. d + new_dqs);
  2585. writel(0, &sdr_scc_mgr->update);
  2586. if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
  2587. PASS_ALL_BITS, &bit_chk,
  2588. 0)) {
  2589. /* USE Set current end of the window */
  2590. end_curr = d;
  2591. /*
  2592. * If a beginning edge of our window has not been seen
  2593. * this is our current begin of the DM window.
  2594. */
  2595. if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
  2596. bgn_curr = d;
  2597. /*
  2598. * If current window is bigger than best seen. Set best
  2599. * seen to be current window.
  2600. */
  2601. if ((end_curr-bgn_curr+1) > win_best) {
  2602. win_best = end_curr-bgn_curr+1;
  2603. bgn_best = bgn_curr;
  2604. end_best = end_curr;
  2605. }
  2606. } else {
  2607. /* We just saw a failing test. Reset temp edge */
  2608. bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
  2609. end_curr = IO_IO_OUT1_DELAY_MAX + 1;
  2610. /* Early exit optimization: if ther remaining delay
  2611. chain space is less than already seen largest window
  2612. we can exit */
  2613. if ((win_best-1) >
  2614. (IO_IO_OUT1_DELAY_MAX - new_dqs - d)) {
  2615. break;
  2616. }
  2617. }
  2618. }
  2619. /* assign left and right edge for cal and reporting; */
  2620. left_edge[0] = -1*bgn_best;
  2621. right_edge[0] = end_best;
  2622. debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d\n", __func__,
  2623. __LINE__, left_edge[0], right_edge[0]);
  2624. /* Move DQS (back to orig) */
  2625. scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
  2626. /* Move DM */
  2627. /* Find middle of window for the DM bit */
  2628. mid = (left_edge[0] - right_edge[0]) / 2;
  2629. /* only move right, since we are not moving DQS/DQ */
  2630. if (mid < 0)
  2631. mid = 0;
  2632. /* dm_marign should fail if we never find a window */
  2633. if (win_best == 0)
  2634. dm_margin = -1;
  2635. else
  2636. dm_margin = left_edge[0] - mid;
  2637. scc_mgr_apply_group_dm_out1_delay(mid);
  2638. writel(0, &sdr_scc_mgr->update);
  2639. debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d mid=%d \
  2640. dm_margin=%d\n", __func__, __LINE__, left_edge[0],
  2641. right_edge[0], mid, dm_margin);
  2642. /* Export values */
  2643. gbl->fom_out += dq_margin + dqs_margin;
  2644. debug_cond(DLEVEL == 2, "%s:%d write_center: dq_margin=%d \
  2645. dqs_margin=%d dm_margin=%d\n", __func__, __LINE__,
  2646. dq_margin, dqs_margin, dm_margin);
  2647. /*
  2648. * Do not remove this line as it makes sure all of our
  2649. * decisions have been applied.
  2650. */
  2651. writel(0, &sdr_scc_mgr->update);
  2652. return (dq_margin >= 0) && (dqs_margin >= 0) && (dm_margin >= 0);
  2653. }
  2654. /* calibrate the write operations */
  2655. static uint32_t rw_mgr_mem_calibrate_writes(uint32_t rank_bgn, uint32_t g,
  2656. uint32_t test_bgn)
  2657. {
  2658. /* update info for sims */
  2659. debug("%s:%d %u %u\n", __func__, __LINE__, g, test_bgn);
  2660. reg_file_set_stage(CAL_STAGE_WRITES);
  2661. reg_file_set_sub_stage(CAL_SUBSTAGE_WRITES_CENTER);
  2662. reg_file_set_group(g);
  2663. if (!rw_mgr_mem_calibrate_writes_center(rank_bgn, g, test_bgn)) {
  2664. set_failing_group_stage(g, CAL_STAGE_WRITES,
  2665. CAL_SUBSTAGE_WRITES_CENTER);
  2666. return 0;
  2667. }
  2668. return 1;
  2669. }
  2670. /* precharge all banks and activate row 0 in bank "000..." and bank "111..." */
  2671. static void mem_precharge_and_activate(void)
  2672. {
  2673. uint32_t r;
  2674. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
  2675. if (param->skip_ranks[r]) {
  2676. /* request to skip the rank */
  2677. continue;
  2678. }
  2679. /* set rank */
  2680. set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
  2681. /* precharge all banks ... */
  2682. writel(RW_MGR_PRECHARGE_ALL, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  2683. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  2684. writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr0);
  2685. writel(RW_MGR_ACTIVATE_0_AND_1_WAIT1,
  2686. &sdr_rw_load_jump_mgr_regs->load_jump_add0);
  2687. writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr1);
  2688. writel(RW_MGR_ACTIVATE_0_AND_1_WAIT2,
  2689. &sdr_rw_load_jump_mgr_regs->load_jump_add1);
  2690. /* activate rows */
  2691. writel(RW_MGR_ACTIVATE_0_AND_1, SDR_PHYGRP_RWMGRGRP_ADDRESS |
  2692. RW_MGR_RUN_SINGLE_GROUP_OFFSET);
  2693. }
  2694. }
  2695. /* Configure various memory related parameters. */
  2696. static void mem_config(void)
  2697. {
  2698. uint32_t rlat, wlat;
  2699. uint32_t rw_wl_nop_cycles;
  2700. uint32_t max_latency;
  2701. debug("%s:%d\n", __func__, __LINE__);
  2702. /* read in write and read latency */
  2703. wlat = readl(&data_mgr->t_wl_add);
  2704. wlat += readl(&data_mgr->mem_t_add);
  2705. /* WL for hard phy does not include additive latency */
  2706. /*
  2707. * add addtional write latency to offset the address/command extra
  2708. * clock cycle. We change the AC mux setting causing AC to be delayed
  2709. * by one mem clock cycle. Only do this for DDR3
  2710. */
  2711. wlat = wlat + 1;
  2712. rlat = readl(&data_mgr->t_rl_add);
  2713. rw_wl_nop_cycles = wlat - 2;
  2714. gbl->rw_wl_nop_cycles = rw_wl_nop_cycles;
  2715. /*
  2716. * For AV/CV, lfifo is hardened and always runs at full rate so
  2717. * max latency in AFI clocks, used here, is correspondingly smaller.
  2718. */
  2719. max_latency = (1<<MAX_LATENCY_COUNT_WIDTH)/1 - 1;
  2720. /* configure for a burst length of 8 */
  2721. /* write latency */
  2722. /* Adjust Write Latency for Hard PHY */
  2723. wlat = wlat + 1;
  2724. /* set a pretty high read latency initially */
  2725. gbl->curr_read_lat = rlat + 16;
  2726. if (gbl->curr_read_lat > max_latency)
  2727. gbl->curr_read_lat = max_latency;
  2728. writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
  2729. /* advertise write latency */
  2730. gbl->curr_write_lat = wlat;
  2731. writel(wlat - 2, &phy_mgr_cfg->afi_wlat);
  2732. /* initialize bit slips */
  2733. mem_precharge_and_activate();
  2734. }
  2735. /* Set VFIFO and LFIFO to instant-on settings in skip calibration mode */
  2736. static void mem_skip_calibrate(void)
  2737. {
  2738. uint32_t vfifo_offset;
  2739. uint32_t i, j, r;
  2740. debug("%s:%d\n", __func__, __LINE__);
  2741. /* Need to update every shadow register set used by the interface */
  2742. for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
  2743. r += NUM_RANKS_PER_SHADOW_REG) {
  2744. /*
  2745. * Set output phase alignment settings appropriate for
  2746. * skip calibration.
  2747. */
  2748. for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
  2749. scc_mgr_set_dqs_en_phase(i, 0);
  2750. #if IO_DLL_CHAIN_LENGTH == 6
  2751. scc_mgr_set_dqdqs_output_phase(i, 6);
  2752. #else
  2753. scc_mgr_set_dqdqs_output_phase(i, 7);
  2754. #endif
  2755. /*
  2756. * Case:33398
  2757. *
  2758. * Write data arrives to the I/O two cycles before write
  2759. * latency is reached (720 deg).
  2760. * -> due to bit-slip in a/c bus
  2761. * -> to allow board skew where dqs is longer than ck
  2762. * -> how often can this happen!?
  2763. * -> can claim back some ptaps for high freq
  2764. * support if we can relax this, but i digress...
  2765. *
  2766. * The write_clk leads mem_ck by 90 deg
  2767. * The minimum ptap of the OPA is 180 deg
  2768. * Each ptap has (360 / IO_DLL_CHAIN_LENGH) deg of delay
  2769. * The write_clk is always delayed by 2 ptaps
  2770. *
  2771. * Hence, to make DQS aligned to CK, we need to delay
  2772. * DQS by:
  2773. * (720 - 90 - 180 - 2 * (360 / IO_DLL_CHAIN_LENGTH))
  2774. *
  2775. * Dividing the above by (360 / IO_DLL_CHAIN_LENGTH)
  2776. * gives us the number of ptaps, which simplies to:
  2777. *
  2778. * (1.25 * IO_DLL_CHAIN_LENGTH - 2)
  2779. */
  2780. scc_mgr_set_dqdqs_output_phase(i, (1.25 *
  2781. IO_DLL_CHAIN_LENGTH - 2));
  2782. }
  2783. writel(0xff, &sdr_scc_mgr->dqs_ena);
  2784. writel(0xff, &sdr_scc_mgr->dqs_io_ena);
  2785. for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
  2786. writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
  2787. SCC_MGR_GROUP_COUNTER_OFFSET);
  2788. }
  2789. writel(0xff, &sdr_scc_mgr->dq_ena);
  2790. writel(0xff, &sdr_scc_mgr->dm_ena);
  2791. writel(0, &sdr_scc_mgr->update);
  2792. }
  2793. /* Compensate for simulation model behaviour */
  2794. for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
  2795. scc_mgr_set_dqs_bus_in_delay(i, 10);
  2796. scc_mgr_load_dqs(i);
  2797. }
  2798. writel(0, &sdr_scc_mgr->update);
  2799. /*
  2800. * ArriaV has hard FIFOs that can only be initialized by incrementing
  2801. * in sequencer.
  2802. */
  2803. vfifo_offset = CALIB_VFIFO_OFFSET;
  2804. for (j = 0; j < vfifo_offset; j++) {
  2805. writel(0xff, &phy_mgr_cmd->inc_vfifo_hard_phy);
  2806. }
  2807. writel(0, &phy_mgr_cmd->fifo_reset);
  2808. /*
  2809. * For ACV with hard lfifo, we get the skip-cal setting from
  2810. * generation-time constant.
  2811. */
  2812. gbl->curr_read_lat = CALIB_LFIFO_OFFSET;
  2813. writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
  2814. }
  2815. /* Memory calibration entry point */
  2816. static uint32_t mem_calibrate(void)
  2817. {
  2818. uint32_t i;
  2819. uint32_t rank_bgn, sr;
  2820. uint32_t write_group, write_test_bgn;
  2821. uint32_t read_group, read_test_bgn;
  2822. uint32_t run_groups, current_run;
  2823. uint32_t failing_groups = 0;
  2824. uint32_t group_failed = 0;
  2825. uint32_t sr_failed = 0;
  2826. debug("%s:%d\n", __func__, __LINE__);
  2827. /* Initialize the data settings */
  2828. gbl->error_substage = CAL_SUBSTAGE_NIL;
  2829. gbl->error_stage = CAL_STAGE_NIL;
  2830. gbl->error_group = 0xff;
  2831. gbl->fom_in = 0;
  2832. gbl->fom_out = 0;
  2833. mem_config();
  2834. for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
  2835. writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
  2836. SCC_MGR_GROUP_COUNTER_OFFSET);
  2837. /* Only needed once to set all groups, pins, DQ, DQS, DM. */
  2838. if (i == 0)
  2839. scc_mgr_set_hhp_extras();
  2840. scc_set_bypass_mode(i);
  2841. }
  2842. if ((dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) {
  2843. /*
  2844. * Set VFIFO and LFIFO to instant-on settings in skip
  2845. * calibration mode.
  2846. */
  2847. mem_skip_calibrate();
  2848. } else {
  2849. for (i = 0; i < NUM_CALIB_REPEAT; i++) {
  2850. /*
  2851. * Zero all delay chain/phase settings for all
  2852. * groups and all shadow register sets.
  2853. */
  2854. scc_mgr_zero_all();
  2855. run_groups = ~param->skip_groups;
  2856. for (write_group = 0, write_test_bgn = 0; write_group
  2857. < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; write_group++,
  2858. write_test_bgn += RW_MGR_MEM_DQ_PER_WRITE_DQS) {
  2859. /* Initialized the group failure */
  2860. group_failed = 0;
  2861. current_run = run_groups & ((1 <<
  2862. RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1);
  2863. run_groups = run_groups >>
  2864. RW_MGR_NUM_DQS_PER_WRITE_GROUP;
  2865. if (current_run == 0)
  2866. continue;
  2867. writel(write_group, SDR_PHYGRP_SCCGRP_ADDRESS |
  2868. SCC_MGR_GROUP_COUNTER_OFFSET);
  2869. scc_mgr_zero_group(write_group, 0);
  2870. for (read_group = write_group *
  2871. RW_MGR_MEM_IF_READ_DQS_WIDTH /
  2872. RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
  2873. read_test_bgn = 0;
  2874. read_group < (write_group + 1) *
  2875. RW_MGR_MEM_IF_READ_DQS_WIDTH /
  2876. RW_MGR_MEM_IF_WRITE_DQS_WIDTH &&
  2877. group_failed == 0;
  2878. read_group++, read_test_bgn +=
  2879. RW_MGR_MEM_DQ_PER_READ_DQS) {
  2880. /* Calibrate the VFIFO */
  2881. if (!((STATIC_CALIB_STEPS) &
  2882. CALIB_SKIP_VFIFO)) {
  2883. if (!rw_mgr_mem_calibrate_vfifo
  2884. (read_group,
  2885. read_test_bgn)) {
  2886. group_failed = 1;
  2887. if (!(gbl->
  2888. phy_debug_mode_flags &
  2889. PHY_DEBUG_SWEEP_ALL_GROUPS)) {
  2890. return 0;
  2891. }
  2892. }
  2893. }
  2894. }
  2895. /* Calibrate the output side */
  2896. if (group_failed == 0) {
  2897. for (rank_bgn = 0, sr = 0; rank_bgn
  2898. < RW_MGR_MEM_NUMBER_OF_RANKS;
  2899. rank_bgn +=
  2900. NUM_RANKS_PER_SHADOW_REG,
  2901. ++sr) {
  2902. sr_failed = 0;
  2903. if (!((STATIC_CALIB_STEPS) &
  2904. CALIB_SKIP_WRITES)) {
  2905. if ((STATIC_CALIB_STEPS)
  2906. & CALIB_SKIP_DELAY_SWEEPS) {
  2907. /* not needed in quick mode! */
  2908. } else {
  2909. /*
  2910. * Determine if this set of
  2911. * ranks should be skipped
  2912. * entirely.
  2913. */
  2914. if (!param->skip_shadow_regs[sr]) {
  2915. if (!rw_mgr_mem_calibrate_writes
  2916. (rank_bgn, write_group,
  2917. write_test_bgn)) {
  2918. sr_failed = 1;
  2919. if (!(gbl->
  2920. phy_debug_mode_flags &
  2921. PHY_DEBUG_SWEEP_ALL_GROUPS)) {
  2922. return 0;
  2923. }
  2924. }
  2925. }
  2926. }
  2927. }
  2928. if (sr_failed != 0)
  2929. group_failed = 1;
  2930. }
  2931. }
  2932. if (group_failed == 0) {
  2933. for (read_group = write_group *
  2934. RW_MGR_MEM_IF_READ_DQS_WIDTH /
  2935. RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
  2936. read_test_bgn = 0;
  2937. read_group < (write_group + 1)
  2938. * RW_MGR_MEM_IF_READ_DQS_WIDTH
  2939. / RW_MGR_MEM_IF_WRITE_DQS_WIDTH &&
  2940. group_failed == 0;
  2941. read_group++, read_test_bgn +=
  2942. RW_MGR_MEM_DQ_PER_READ_DQS) {
  2943. if (!((STATIC_CALIB_STEPS) &
  2944. CALIB_SKIP_WRITES)) {
  2945. if (!rw_mgr_mem_calibrate_vfifo_end
  2946. (read_group, read_test_bgn)) {
  2947. group_failed = 1;
  2948. if (!(gbl->phy_debug_mode_flags
  2949. & PHY_DEBUG_SWEEP_ALL_GROUPS)) {
  2950. return 0;
  2951. }
  2952. }
  2953. }
  2954. }
  2955. }
  2956. if (group_failed != 0)
  2957. failing_groups++;
  2958. }
  2959. /*
  2960. * USER If there are any failing groups then report
  2961. * the failure.
  2962. */
  2963. if (failing_groups != 0)
  2964. return 0;
  2965. /* Calibrate the LFIFO */
  2966. if (!((STATIC_CALIB_STEPS) & CALIB_SKIP_LFIFO)) {
  2967. /*
  2968. * If we're skipping groups as part of debug,
  2969. * don't calibrate LFIFO.
  2970. */
  2971. if (param->skip_groups == 0) {
  2972. if (!rw_mgr_mem_calibrate_lfifo())
  2973. return 0;
  2974. }
  2975. }
  2976. }
  2977. }
  2978. /*
  2979. * Do not remove this line as it makes sure all of our decisions
  2980. * have been applied.
  2981. */
  2982. writel(0, &sdr_scc_mgr->update);
  2983. return 1;
  2984. }
  2985. static uint32_t run_mem_calibrate(void)
  2986. {
  2987. uint32_t pass;
  2988. uint32_t debug_info;
  2989. debug("%s:%d\n", __func__, __LINE__);
  2990. /* Reset pass/fail status shown on afi_cal_success/fail */
  2991. writel(PHY_MGR_CAL_RESET, &phy_mgr_cfg->cal_status);
  2992. /* stop tracking manger */
  2993. uint32_t ctrlcfg = readl(&sdr_ctrl->ctrl_cfg);
  2994. writel(ctrlcfg & 0xFFBFFFFF, &sdr_ctrl->ctrl_cfg);
  2995. initialize();
  2996. rw_mgr_mem_initialize();
  2997. pass = mem_calibrate();
  2998. mem_precharge_and_activate();
  2999. writel(0, &phy_mgr_cmd->fifo_reset);
  3000. /*
  3001. * Handoff:
  3002. * Don't return control of the PHY back to AFI when in debug mode.
  3003. */
  3004. if ((gbl->phy_debug_mode_flags & PHY_DEBUG_IN_DEBUG_MODE) == 0) {
  3005. rw_mgr_mem_handoff();
  3006. /*
  3007. * In Hard PHY this is a 2-bit control:
  3008. * 0: AFI Mux Select
  3009. * 1: DDIO Mux Select
  3010. */
  3011. writel(0x2, &phy_mgr_cfg->mux_sel);
  3012. }
  3013. writel(ctrlcfg, &sdr_ctrl->ctrl_cfg);
  3014. if (pass) {
  3015. printf("%s: CALIBRATION PASSED\n", __FILE__);
  3016. gbl->fom_in /= 2;
  3017. gbl->fom_out /= 2;
  3018. if (gbl->fom_in > 0xff)
  3019. gbl->fom_in = 0xff;
  3020. if (gbl->fom_out > 0xff)
  3021. gbl->fom_out = 0xff;
  3022. /* Update the FOM in the register file */
  3023. debug_info = gbl->fom_in;
  3024. debug_info |= gbl->fom_out << 8;
  3025. writel(debug_info, &sdr_reg_file->fom);
  3026. writel(debug_info, &phy_mgr_cfg->cal_debug_info);
  3027. writel(PHY_MGR_CAL_SUCCESS, &phy_mgr_cfg->cal_status);
  3028. } else {
  3029. printf("%s: CALIBRATION FAILED\n", __FILE__);
  3030. debug_info = gbl->error_stage;
  3031. debug_info |= gbl->error_substage << 8;
  3032. debug_info |= gbl->error_group << 16;
  3033. writel(debug_info, &sdr_reg_file->failing_stage);
  3034. writel(debug_info, &phy_mgr_cfg->cal_debug_info);
  3035. writel(PHY_MGR_CAL_FAIL, &phy_mgr_cfg->cal_status);
  3036. /* Update the failing group/stage in the register file */
  3037. debug_info = gbl->error_stage;
  3038. debug_info |= gbl->error_substage << 8;
  3039. debug_info |= gbl->error_group << 16;
  3040. writel(debug_info, &sdr_reg_file->failing_stage);
  3041. }
  3042. return pass;
  3043. }
  3044. /**
  3045. * hc_initialize_rom_data() - Initialize ROM data
  3046. *
  3047. * Initialize ROM data.
  3048. */
  3049. static void hc_initialize_rom_data(void)
  3050. {
  3051. u32 i, addr;
  3052. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_INST_ROM_WRITE_OFFSET;
  3053. for (i = 0; i < ARRAY_SIZE(inst_rom_init); i++)
  3054. writel(inst_rom_init[i], addr + (i << 2));
  3055. addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_AC_ROM_WRITE_OFFSET;
  3056. for (i = 0; i < ARRAY_SIZE(ac_rom_init); i++)
  3057. writel(ac_rom_init[i], addr + (i << 2));
  3058. }
  3059. /**
  3060. * initialize_reg_file() - Initialize SDR register file
  3061. *
  3062. * Initialize SDR register file.
  3063. */
  3064. static void initialize_reg_file(void)
  3065. {
  3066. /* Initialize the register file with the correct data */
  3067. writel(REG_FILE_INIT_SEQ_SIGNATURE, &sdr_reg_file->signature);
  3068. writel(0, &sdr_reg_file->debug_data_addr);
  3069. writel(0, &sdr_reg_file->cur_stage);
  3070. writel(0, &sdr_reg_file->fom);
  3071. writel(0, &sdr_reg_file->failing_stage);
  3072. writel(0, &sdr_reg_file->debug1);
  3073. writel(0, &sdr_reg_file->debug2);
  3074. }
  3075. /**
  3076. * initialize_hps_phy() - Initialize HPS PHY
  3077. *
  3078. * Initialize HPS PHY.
  3079. */
  3080. static void initialize_hps_phy(void)
  3081. {
  3082. uint32_t reg;
  3083. /*
  3084. * Tracking also gets configured here because it's in the
  3085. * same register.
  3086. */
  3087. uint32_t trk_sample_count = 7500;
  3088. uint32_t trk_long_idle_sample_count = (10 << 16) | 100;
  3089. /*
  3090. * Format is number of outer loops in the 16 MSB, sample
  3091. * count in 16 LSB.
  3092. */
  3093. reg = 0;
  3094. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(2);
  3095. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQDELAYEN_SET(1);
  3096. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSDELAYEN_SET(1);
  3097. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSLOGICDELAYEN_SET(1);
  3098. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_RESETDELAYEN_SET(0);
  3099. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(1);
  3100. /*
  3101. * This field selects the intrinsic latency to RDATA_EN/FULL path.
  3102. * 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles.
  3103. */
  3104. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ADDLATSEL_SET(0);
  3105. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_SET(
  3106. trk_sample_count);
  3107. writel(reg, &sdr_ctrl->phy_ctrl0);
  3108. reg = 0;
  3109. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_SAMPLECOUNT_31_20_SET(
  3110. trk_sample_count >>
  3111. SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_WIDTH);
  3112. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_SET(
  3113. trk_long_idle_sample_count);
  3114. writel(reg, &sdr_ctrl->phy_ctrl1);
  3115. reg = 0;
  3116. reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_LONGIDLESAMPLECOUNT_31_20_SET(
  3117. trk_long_idle_sample_count >>
  3118. SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_WIDTH);
  3119. writel(reg, &sdr_ctrl->phy_ctrl2);
  3120. }
  3121. static void initialize_tracking(void)
  3122. {
  3123. uint32_t concatenated_longidle = 0x0;
  3124. uint32_t concatenated_delays = 0x0;
  3125. uint32_t concatenated_rw_addr = 0x0;
  3126. uint32_t concatenated_refresh = 0x0;
  3127. uint32_t trk_sample_count = 7500;
  3128. uint32_t dtaps_per_ptap;
  3129. uint32_t tmp_delay;
  3130. /*
  3131. * compute usable version of value in case we skip full
  3132. * computation later
  3133. */
  3134. dtaps_per_ptap = 0;
  3135. tmp_delay = 0;
  3136. while (tmp_delay < IO_DELAY_PER_OPA_TAP) {
  3137. dtaps_per_ptap++;
  3138. tmp_delay += IO_DELAY_PER_DCHAIN_TAP;
  3139. }
  3140. dtaps_per_ptap--;
  3141. concatenated_longidle = concatenated_longidle ^ 10;
  3142. /*longidle outer loop */
  3143. concatenated_longidle = concatenated_longidle << 16;
  3144. concatenated_longidle = concatenated_longidle ^ 100;
  3145. /*longidle sample count */
  3146. concatenated_delays = concatenated_delays ^ 243;
  3147. /* trfc, worst case of 933Mhz 4Gb */
  3148. concatenated_delays = concatenated_delays << 8;
  3149. concatenated_delays = concatenated_delays ^ 14;
  3150. /* trcd, worst case */
  3151. concatenated_delays = concatenated_delays << 8;
  3152. concatenated_delays = concatenated_delays ^ 10;
  3153. /* vfifo wait */
  3154. concatenated_delays = concatenated_delays << 8;
  3155. concatenated_delays = concatenated_delays ^ 4;
  3156. /* mux delay */
  3157. concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_IDLE;
  3158. concatenated_rw_addr = concatenated_rw_addr << 8;
  3159. concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_ACTIVATE_1;
  3160. concatenated_rw_addr = concatenated_rw_addr << 8;
  3161. concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_SGLE_READ;
  3162. concatenated_rw_addr = concatenated_rw_addr << 8;
  3163. concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_PRECHARGE_ALL;
  3164. concatenated_refresh = concatenated_refresh ^ RW_MGR_REFRESH_ALL;
  3165. concatenated_refresh = concatenated_refresh << 24;
  3166. concatenated_refresh = concatenated_refresh ^ 1000; /* trefi */
  3167. /* Initialize the register file with the correct data */
  3168. writel(dtaps_per_ptap, &sdr_reg_file->dtaps_per_ptap);
  3169. writel(trk_sample_count, &sdr_reg_file->trk_sample_count);
  3170. writel(concatenated_longidle, &sdr_reg_file->trk_longidle);
  3171. writel(concatenated_delays, &sdr_reg_file->delays);
  3172. writel(concatenated_rw_addr, &sdr_reg_file->trk_rw_mgr_addr);
  3173. writel(RW_MGR_MEM_IF_READ_DQS_WIDTH, &sdr_reg_file->trk_read_dqs_width);
  3174. writel(concatenated_refresh, &sdr_reg_file->trk_rfsh);
  3175. }
  3176. int sdram_calibration_full(void)
  3177. {
  3178. struct param_type my_param;
  3179. struct gbl_type my_gbl;
  3180. uint32_t pass;
  3181. uint32_t i;
  3182. param = &my_param;
  3183. gbl = &my_gbl;
  3184. /* Initialize the debug mode flags */
  3185. gbl->phy_debug_mode_flags = 0;
  3186. /* Set the calibration enabled by default */
  3187. gbl->phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT;
  3188. /*
  3189. * Only sweep all groups (regardless of fail state) by default
  3190. * Set enabled read test by default.
  3191. */
  3192. #if DISABLE_GUARANTEED_READ
  3193. gbl->phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ;
  3194. #endif
  3195. /* Initialize the register file */
  3196. initialize_reg_file();
  3197. /* Initialize any PHY CSR */
  3198. initialize_hps_phy();
  3199. scc_mgr_initialize();
  3200. initialize_tracking();
  3201. /* USER Enable all ranks, groups */
  3202. for (i = 0; i < RW_MGR_MEM_NUMBER_OF_RANKS; i++)
  3203. param->skip_ranks[i] = 0;
  3204. for (i = 0; i < NUM_SHADOW_REGS; ++i)
  3205. param->skip_shadow_regs[i] = 0;
  3206. param->skip_groups = 0;
  3207. printf("%s: Preparing to start memory calibration\n", __FILE__);
  3208. debug("%s:%d\n", __func__, __LINE__);
  3209. debug_cond(DLEVEL == 1,
  3210. "DDR3 FULL_RATE ranks=%u cs/dimm=%u dq/dqs=%u,%u vg/dqs=%u,%u ",
  3211. RW_MGR_MEM_NUMBER_OF_RANKS, RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM,
  3212. RW_MGR_MEM_DQ_PER_READ_DQS, RW_MGR_MEM_DQ_PER_WRITE_DQS,
  3213. RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS,
  3214. RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
  3215. debug_cond(DLEVEL == 1,
  3216. "dqs=%u,%u dq=%u dm=%u ptap_delay=%u dtap_delay=%u ",
  3217. RW_MGR_MEM_IF_READ_DQS_WIDTH, RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
  3218. RW_MGR_MEM_DATA_WIDTH, RW_MGR_MEM_DATA_MASK_WIDTH,
  3219. IO_DELAY_PER_OPA_TAP, IO_DELAY_PER_DCHAIN_TAP);
  3220. debug_cond(DLEVEL == 1, "dtap_dqsen_delay=%u, dll=%u",
  3221. IO_DELAY_PER_DQS_EN_DCHAIN_TAP, IO_DLL_CHAIN_LENGTH);
  3222. debug_cond(DLEVEL == 1, "max values: en_p=%u dqdqs_p=%u en_d=%u dqs_in_d=%u ",
  3223. IO_DQS_EN_PHASE_MAX, IO_DQDQS_OUT_PHASE_MAX,
  3224. IO_DQS_EN_DELAY_MAX, IO_DQS_IN_DELAY_MAX);
  3225. debug_cond(DLEVEL == 1, "io_in_d=%u io_out1_d=%u io_out2_d=%u ",
  3226. IO_IO_IN_DELAY_MAX, IO_IO_OUT1_DELAY_MAX,
  3227. IO_IO_OUT2_DELAY_MAX);
  3228. debug_cond(DLEVEL == 1, "dqs_in_reserve=%u dqs_out_reserve=%u\n",
  3229. IO_DQS_IN_RESERVE, IO_DQS_OUT_RESERVE);
  3230. hc_initialize_rom_data();
  3231. /* update info for sims */
  3232. reg_file_set_stage(CAL_STAGE_NIL);
  3233. reg_file_set_group(0);
  3234. /*
  3235. * Load global needed for those actions that require
  3236. * some dynamic calibration support.
  3237. */
  3238. dyn_calib_steps = STATIC_CALIB_STEPS;
  3239. /*
  3240. * Load global to allow dynamic selection of delay loop settings
  3241. * based on calibration mode.
  3242. */
  3243. if (!(dyn_calib_steps & CALIB_SKIP_DELAY_LOOPS))
  3244. skip_delay_mask = 0xff;
  3245. else
  3246. skip_delay_mask = 0x0;
  3247. pass = run_mem_calibrate();
  3248. printf("%s: Calibration complete\n", __FILE__);
  3249. return pass;
  3250. }