Lines Matching refs:seq
57 ((non_skip_value) & seq->skip_delay_mask)
74 static void set_failing_group_stage(struct socfpga_sdrseq *seq, in set_failing_group_stage() argument
81 if (seq->gbl.error_stage == CAL_STAGE_NIL) { in set_failing_group_stage()
82 seq->gbl.error_substage = substage; in set_failing_group_stage()
83 seq->gbl.error_stage = stage; in set_failing_group_stage()
84 seq->gbl.error_group = group; in set_failing_group_stage()
109 static void phy_mgr_initialize(struct socfpga_sdrseq *seq) in phy_mgr_initialize() argument
131 if ((seq->dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) in phy_mgr_initialize()
134 ratio = seq->rwcfg->mem_dq_per_read_dqs / in phy_mgr_initialize()
135 seq->rwcfg->mem_virtual_groups_per_read_dqs; in phy_mgr_initialize()
136 seq->param.read_correct_mask_vg = (1 << ratio) - 1; in phy_mgr_initialize()
137 seq->param.write_correct_mask_vg = (1 << ratio) - 1; in phy_mgr_initialize()
138 seq->param.read_correct_mask = (1 << seq->rwcfg->mem_dq_per_read_dqs) in phy_mgr_initialize()
140 seq->param.write_correct_mask = (1 << seq->rwcfg->mem_dq_per_write_dqs) in phy_mgr_initialize()
151 static void set_rank_and_odt_mask(struct socfpga_sdrseq *seq, in set_rank_and_odt_mask() argument
162 switch (seq->rwcfg->mem_number_of_ranks) { in set_rank_and_odt_mask()
169 if (seq->rwcfg->mem_number_of_cs_per_dimm == 1) { in set_rank_and_odt_mask()
324 static void scc_mgr_set_dqs_io_in_delay(struct socfpga_sdrseq *seq, in scc_mgr_set_dqs_io_in_delay() argument
328 seq->rwcfg->mem_dq_per_write_dqs, delay); in scc_mgr_set_dqs_io_in_delay()
331 static void scc_mgr_set_dm_in_delay(struct socfpga_sdrseq *seq, u32 dm, in scc_mgr_set_dm_in_delay() argument
335 seq->rwcfg->mem_dq_per_write_dqs + 1 + dm, in scc_mgr_set_dm_in_delay()
344 static void scc_mgr_set_dqs_out1_delay(struct socfpga_sdrseq *seq, in scc_mgr_set_dqs_out1_delay() argument
348 seq->rwcfg->mem_dq_per_write_dqs, delay); in scc_mgr_set_dqs_out1_delay()
351 static void scc_mgr_set_dm_out1_delay(struct socfpga_sdrseq *seq, u32 dm, in scc_mgr_set_dm_out1_delay() argument
355 seq->rwcfg->mem_dq_per_write_dqs + 1 + dm, in scc_mgr_set_dm_out1_delay()
393 static void scc_mgr_set_all_ranks(struct socfpga_sdrseq *seq, in scc_mgr_set_all_ranks() argument
399 for (r = 0; r < seq->rwcfg->mem_number_of_ranks; in scc_mgr_set_all_ranks()
410 static void scc_mgr_set_dqs_en_phase_all_ranks(struct socfpga_sdrseq *seq, in scc_mgr_set_dqs_en_phase_all_ranks() argument
421 scc_mgr_set_all_ranks(seq, SCC_MGR_DQS_EN_PHASE_OFFSET, in scc_mgr_set_dqs_en_phase_all_ranks()
425 static void scc_mgr_set_dqdqs_output_phase_all_ranks(struct socfpga_sdrseq *seq, in scc_mgr_set_dqdqs_output_phase_all_ranks() argument
436 scc_mgr_set_all_ranks(seq, SCC_MGR_DQDQS_OUT_PHASE_OFFSET, in scc_mgr_set_dqdqs_output_phase_all_ranks()
440 static void scc_mgr_set_dqs_en_delay_all_ranks(struct socfpga_sdrseq *seq, in scc_mgr_set_dqs_en_delay_all_ranks() argument
451 scc_mgr_set_all_ranks(seq, SCC_MGR_DQS_EN_DELAY_OFFSET, in scc_mgr_set_dqs_en_delay_all_ranks()
462 static void scc_mgr_set_oct_out1_delay(struct socfpga_sdrseq *seq, in scc_mgr_set_oct_out1_delay() argument
465 const int ratio = seq->rwcfg->mem_if_read_dqs_width / in scc_mgr_set_oct_out1_delay()
466 seq->rwcfg->mem_if_write_dqs_width; in scc_mgr_set_oct_out1_delay()
514 static void scc_mgr_zero_all(struct socfpga_sdrseq *seq) in scc_mgr_zero_all() argument
522 for (r = 0; r < seq->rwcfg->mem_number_of_ranks; in scc_mgr_zero_all()
524 for (i = 0; i < seq->rwcfg->mem_if_read_dqs_width; i++) { in scc_mgr_zero_all()
531 seq->iocfg->dqs_in_reserve in scc_mgr_zero_all()
537 for (i = 0; i < seq->rwcfg->mem_if_write_dqs_width; i++) { in scc_mgr_zero_all()
540 scc_mgr_set_oct_out1_delay(seq, i, in scc_mgr_zero_all()
541 seq->iocfg->dqs_out_reserve); in scc_mgr_zero_all()
578 static void scc_mgr_load_dqs_for_write_group(struct socfpga_sdrseq *seq, in scc_mgr_load_dqs_for_write_group() argument
581 const int ratio = seq->rwcfg->mem_if_read_dqs_width / in scc_mgr_load_dqs_for_write_group()
582 seq->rwcfg->mem_if_write_dqs_width; in scc_mgr_load_dqs_for_write_group()
601 static void scc_mgr_zero_group(struct socfpga_sdrseq *seq, in scc_mgr_zero_group() argument
606 for (r = 0; r < seq->rwcfg->mem_number_of_ranks; in scc_mgr_zero_group()
609 for (i = 0; i < seq->rwcfg->mem_dq_per_write_dqs; i++) { in scc_mgr_zero_group()
621 scc_mgr_set_dm_in_delay(seq, i, 0); in scc_mgr_zero_group()
622 scc_mgr_set_dm_out1_delay(seq, i, 0); in scc_mgr_zero_group()
630 scc_mgr_set_dqs_io_in_delay(seq, 0); in scc_mgr_zero_group()
633 scc_mgr_set_dqs_out1_delay(seq, seq->iocfg->dqs_out_reserve); in scc_mgr_zero_group()
634 scc_mgr_set_oct_out1_delay(seq, write_group, in scc_mgr_zero_group()
635 seq->iocfg->dqs_out_reserve); in scc_mgr_zero_group()
636 scc_mgr_load_dqs_for_write_group(seq, write_group); in scc_mgr_zero_group()
650 static void scc_mgr_apply_group_dq_in_delay(struct socfpga_sdrseq *seq, in scc_mgr_apply_group_dq_in_delay() argument
655 for (i = 0, p = group_bgn; i < seq->rwcfg->mem_dq_per_read_dqs; in scc_mgr_apply_group_dq_in_delay()
669 static void scc_mgr_apply_group_dq_out1_delay(struct socfpga_sdrseq *seq, in scc_mgr_apply_group_dq_out1_delay() argument
674 for (i = 0; i < seq->rwcfg->mem_dq_per_write_dqs; i++) { in scc_mgr_apply_group_dq_out1_delay()
681 static void scc_mgr_apply_group_dm_out1_delay(struct socfpga_sdrseq *seq, in scc_mgr_apply_group_dm_out1_delay() argument
687 scc_mgr_set_dm_out1_delay(seq, i, delay1); in scc_mgr_apply_group_dm_out1_delay()
694 static void scc_mgr_apply_group_dqs_io_and_oct_out1(struct socfpga_sdrseq *seq, in scc_mgr_apply_group_dqs_io_and_oct_out1() argument
697 scc_mgr_set_dqs_out1_delay(seq, delay); in scc_mgr_apply_group_dqs_io_and_oct_out1()
700 scc_mgr_set_oct_out1_delay(seq, write_group, delay); in scc_mgr_apply_group_dqs_io_and_oct_out1()
701 scc_mgr_load_dqs_for_write_group(seq, write_group); in scc_mgr_apply_group_dqs_io_and_oct_out1()
712 static void scc_mgr_apply_group_all_out_delay_add(struct socfpga_sdrseq *seq, in scc_mgr_apply_group_all_out_delay_add() argument
719 for (i = 0; i < seq->rwcfg->mem_dq_per_write_dqs; i++) in scc_mgr_apply_group_all_out_delay_add()
728 if (new_delay > seq->iocfg->io_out2_delay_max) { in scc_mgr_apply_group_all_out_delay_add()
732 seq->iocfg->io_out2_delay_max, in scc_mgr_apply_group_all_out_delay_add()
733 new_delay - seq->iocfg->io_out2_delay_max); in scc_mgr_apply_group_all_out_delay_add()
734 new_delay -= seq->iocfg->io_out2_delay_max; in scc_mgr_apply_group_all_out_delay_add()
735 scc_mgr_set_dqs_out1_delay(seq, new_delay); in scc_mgr_apply_group_all_out_delay_add()
742 if (new_delay > seq->iocfg->io_out2_delay_max) { in scc_mgr_apply_group_all_out_delay_add()
746 new_delay, seq->iocfg->io_out2_delay_max, in scc_mgr_apply_group_all_out_delay_add()
747 new_delay - seq->iocfg->io_out2_delay_max); in scc_mgr_apply_group_all_out_delay_add()
748 new_delay -= seq->iocfg->io_out2_delay_max; in scc_mgr_apply_group_all_out_delay_add()
749 scc_mgr_set_oct_out1_delay(seq, write_group, new_delay); in scc_mgr_apply_group_all_out_delay_add()
752 scc_mgr_load_dqs_for_write_group(seq, write_group); in scc_mgr_apply_group_all_out_delay_add()
764 scc_mgr_apply_group_all_out_delay_add_all_ranks(struct socfpga_sdrseq *seq, in scc_mgr_apply_group_all_out_delay_add_all_ranks() argument
770 for (r = 0; r < seq->rwcfg->mem_number_of_ranks; in scc_mgr_apply_group_all_out_delay_add_all_ranks()
772 scc_mgr_apply_group_all_out_delay_add(seq, write_group, delay); in scc_mgr_apply_group_all_out_delay_add_all_ranks()
783 static void set_jump_as_return(struct socfpga_sdrseq *seq) in set_jump_as_return() argument
791 writel(seq->rwcfg->rreturn, &sdr_rw_load_jump_mgr_regs->load_jump_add0); in set_jump_as_return()
800 static void delay_for_n_mem_clocks(struct socfpga_sdrseq *seq, in delay_for_n_mem_clocks() argument
811 afi_clocks = DIV_ROUND_UP(clocks, seq->misccfg->afi_rate_ratio); in delay_for_n_mem_clocks()
847 writel(seq->rwcfg->idle_loop1, in delay_for_n_mem_clocks()
850 writel(seq->rwcfg->idle_loop1, SDR_PHYGRP_RWMGRGRP_ADDRESS | in delay_for_n_mem_clocks()
859 writel(seq->rwcfg->idle_loop2, in delay_for_n_mem_clocks()
862 writel(seq->rwcfg->idle_loop2, in delay_for_n_mem_clocks()
866 writel(seq->rwcfg->idle_loop2, in delay_for_n_mem_clocks()
874 static void delay_for_n_ns(struct socfpga_sdrseq *seq, const u32 ns) in delay_for_n_ns() argument
876 delay_for_n_mem_clocks(seq, (ns * seq->misccfg->afi_clk_freq * in delay_for_n_ns()
877 seq->misccfg->afi_rate_ratio) / 1000); in delay_for_n_ns()
889 static void rw_mgr_mem_init_load_regs(struct socfpga_sdrseq *seq, in rw_mgr_mem_init_load_regs() argument
918 static void rw_mgr_mem_load_user_ddr2(struct socfpga_sdrseq *seq, in rw_mgr_mem_load_user_ddr2() argument
925 for (r = 0; r < seq->rwcfg->mem_number_of_ranks; r++) { in rw_mgr_mem_load_user_ddr2()
927 set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_OFF); in rw_mgr_mem_load_user_ddr2()
930 writel(seq->rwcfg->precharge_all, grpaddr); in rw_mgr_mem_load_user_ddr2()
932 writel(seq->rwcfg->emr2, grpaddr); in rw_mgr_mem_load_user_ddr2()
933 writel(seq->rwcfg->emr3, grpaddr); in rw_mgr_mem_load_user_ddr2()
934 writel(seq->rwcfg->emr, grpaddr); in rw_mgr_mem_load_user_ddr2()
937 writel(seq->rwcfg->mr_user, grpaddr); in rw_mgr_mem_load_user_ddr2()
941 writel(seq->rwcfg->mr_dll_reset, grpaddr); in rw_mgr_mem_load_user_ddr2()
943 writel(seq->rwcfg->precharge_all, grpaddr); in rw_mgr_mem_load_user_ddr2()
945 writel(seq->rwcfg->refresh, grpaddr); in rw_mgr_mem_load_user_ddr2()
946 delay_for_n_ns(seq, 200); in rw_mgr_mem_load_user_ddr2()
947 writel(seq->rwcfg->refresh, grpaddr); in rw_mgr_mem_load_user_ddr2()
948 delay_for_n_ns(seq, 200); in rw_mgr_mem_load_user_ddr2()
950 writel(seq->rwcfg->mr_calib, grpaddr); in rw_mgr_mem_load_user_ddr2()
952 writel(seq->rwcfg->emr, grpaddr); in rw_mgr_mem_load_user_ddr2()
953 delay_for_n_mem_clocks(seq, 200); in rw_mgr_mem_load_user_ddr2()
965 static void rw_mgr_mem_load_user_ddr3(struct socfpga_sdrseq *seq, in rw_mgr_mem_load_user_ddr3() argument
973 for (r = 0; r < seq->rwcfg->mem_number_of_ranks; r++) { in rw_mgr_mem_load_user_ddr3()
975 set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_OFF); in rw_mgr_mem_load_user_ddr3()
979 writel(seq->rwcfg->precharge_all, grpaddr); in rw_mgr_mem_load_user_ddr3()
985 if ((seq->rwcfg->mem_address_mirroring >> r) & 0x1) { in rw_mgr_mem_load_user_ddr3()
986 set_jump_as_return(seq); in rw_mgr_mem_load_user_ddr3()
987 writel(seq->rwcfg->mrs2_mirr, grpaddr); in rw_mgr_mem_load_user_ddr3()
988 delay_for_n_mem_clocks(seq, 4); in rw_mgr_mem_load_user_ddr3()
989 set_jump_as_return(seq); in rw_mgr_mem_load_user_ddr3()
990 writel(seq->rwcfg->mrs3_mirr, grpaddr); in rw_mgr_mem_load_user_ddr3()
991 delay_for_n_mem_clocks(seq, 4); in rw_mgr_mem_load_user_ddr3()
992 set_jump_as_return(seq); in rw_mgr_mem_load_user_ddr3()
993 writel(seq->rwcfg->mrs1_mirr, grpaddr); in rw_mgr_mem_load_user_ddr3()
994 delay_for_n_mem_clocks(seq, 4); in rw_mgr_mem_load_user_ddr3()
995 set_jump_as_return(seq); in rw_mgr_mem_load_user_ddr3()
998 set_jump_as_return(seq); in rw_mgr_mem_load_user_ddr3()
999 writel(seq->rwcfg->mrs2, grpaddr); in rw_mgr_mem_load_user_ddr3()
1000 delay_for_n_mem_clocks(seq, 4); in rw_mgr_mem_load_user_ddr3()
1001 set_jump_as_return(seq); in rw_mgr_mem_load_user_ddr3()
1002 writel(seq->rwcfg->mrs3, grpaddr); in rw_mgr_mem_load_user_ddr3()
1003 delay_for_n_mem_clocks(seq, 4); in rw_mgr_mem_load_user_ddr3()
1004 set_jump_as_return(seq); in rw_mgr_mem_load_user_ddr3()
1005 writel(seq->rwcfg->mrs1, grpaddr); in rw_mgr_mem_load_user_ddr3()
1006 set_jump_as_return(seq); in rw_mgr_mem_load_user_ddr3()
1013 set_jump_as_return(seq); in rw_mgr_mem_load_user_ddr3()
1014 writel(seq->rwcfg->zqcl, grpaddr); in rw_mgr_mem_load_user_ddr3()
1017 delay_for_n_mem_clocks(seq, 512); in rw_mgr_mem_load_user_ddr3()
1029 static void rw_mgr_mem_load_user(struct socfpga_sdrseq *seq, in rw_mgr_mem_load_user() argument
1034 rw_mgr_mem_load_user_ddr2(seq, precharge); in rw_mgr_mem_load_user()
1036 rw_mgr_mem_load_user_ddr3(seq, fin1, fin2, precharge); in rw_mgr_mem_load_user()
1045 static void rw_mgr_mem_initialize(struct socfpga_sdrseq *seq) in rw_mgr_mem_initialize() argument
1078 rw_mgr_mem_init_load_regs(seq, seq->misccfg->tinit_cntr0_val, in rw_mgr_mem_initialize()
1079 seq->misccfg->tinit_cntr1_val, in rw_mgr_mem_initialize()
1080 seq->misccfg->tinit_cntr2_val, in rw_mgr_mem_initialize()
1081 seq->rwcfg->init_reset_0_cke_0); in rw_mgr_mem_initialize()
1087 writel(seq->rwcfg->nop, SDR_PHYGRP_RWMGRGRP_ADDRESS | in rw_mgr_mem_initialize()
1093 delay_for_n_ns(seq, 400); in rw_mgr_mem_initialize()
1109 rw_mgr_mem_init_load_regs(seq, seq->misccfg->treset_cntr0_val, in rw_mgr_mem_initialize()
1110 seq->misccfg->treset_cntr1_val, in rw_mgr_mem_initialize()
1111 seq->misccfg->treset_cntr2_val, in rw_mgr_mem_initialize()
1112 seq->rwcfg->init_reset_1_cke_0); in rw_mgr_mem_initialize()
1116 delay_for_n_mem_clocks(seq, 250); in rw_mgr_mem_initialize()
1119 rw_mgr_mem_load_user(seq, seq->rwcfg->mrs0_dll_reset_mirr, in rw_mgr_mem_initialize()
1120 seq->rwcfg->mrs0_dll_reset, 0); in rw_mgr_mem_initialize()
1129 static void rw_mgr_mem_handoff(struct socfpga_sdrseq *seq) in rw_mgr_mem_handoff() argument
1131 rw_mgr_mem_load_user(seq, seq->rwcfg->mrs0_user_mirr, in rw_mgr_mem_handoff()
1132 seq->rwcfg->mrs0_user, 1); in rw_mgr_mem_handoff()
1148 static void rw_mgr_mem_calibrate_write_test_issue(struct socfpga_sdrseq *seq, in rw_mgr_mem_calibrate_write_test_issue() argument
1153 seq->misccfg->enable_super_quick_calibration; in rw_mgr_mem_calibrate_write_test_issue()
1183 rw_wl_nop_cycles = seq->gbl.rw_wl_nop_cycles; in rw_mgr_mem_calibrate_write_test_issue()
1196 mcc_instruction = seq->rwcfg->lfsr_wr_rd_dm_bank_0_wl_1; in rw_mgr_mem_calibrate_write_test_issue()
1197 writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_data, in rw_mgr_mem_calibrate_write_test_issue()
1199 writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_nop, in rw_mgr_mem_calibrate_write_test_issue()
1202 mcc_instruction = seq->rwcfg->lfsr_wr_rd_bank_0_wl_1; in rw_mgr_mem_calibrate_write_test_issue()
1203 writel(seq->rwcfg->lfsr_wr_rd_bank_0_data, in rw_mgr_mem_calibrate_write_test_issue()
1205 writel(seq->rwcfg->lfsr_wr_rd_bank_0_nop, in rw_mgr_mem_calibrate_write_test_issue()
1218 mcc_instruction = seq->rwcfg->lfsr_wr_rd_dm_bank_0; in rw_mgr_mem_calibrate_write_test_issue()
1219 writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_dqs, in rw_mgr_mem_calibrate_write_test_issue()
1222 mcc_instruction = seq->rwcfg->lfsr_wr_rd_bank_0; in rw_mgr_mem_calibrate_write_test_issue()
1223 writel(seq->rwcfg->lfsr_wr_rd_bank_0_dqs, in rw_mgr_mem_calibrate_write_test_issue()
1241 mcc_instruction = seq->rwcfg->lfsr_wr_rd_dm_bank_0; in rw_mgr_mem_calibrate_write_test_issue()
1242 writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_nop, in rw_mgr_mem_calibrate_write_test_issue()
1245 mcc_instruction = seq->rwcfg->lfsr_wr_rd_bank_0; in rw_mgr_mem_calibrate_write_test_issue()
1246 writel(seq->rwcfg->lfsr_wr_rd_bank_0_nop, in rw_mgr_mem_calibrate_write_test_issue()
1268 writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_wait, in rw_mgr_mem_calibrate_write_test_issue()
1271 writel(seq->rwcfg->lfsr_wr_rd_bank_0_wait, in rw_mgr_mem_calibrate_write_test_issue()
1293 rw_mgr_mem_calibrate_write_test(struct socfpga_sdrseq *seq, in rw_mgr_mem_calibrate_write_test() argument
1299 seq->rwcfg->mem_number_of_ranks : in rw_mgr_mem_calibrate_write_test()
1301 const u32 shift_ratio = seq->rwcfg->mem_dq_per_write_dqs / in rw_mgr_mem_calibrate_write_test()
1302 seq->rwcfg->mem_virtual_groups_per_write_dqs; in rw_mgr_mem_calibrate_write_test()
1303 const u32 correct_mask_vg = seq->param.write_correct_mask_vg; in rw_mgr_mem_calibrate_write_test()
1308 *bit_chk = seq->param.write_correct_mask; in rw_mgr_mem_calibrate_write_test()
1312 set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_READ_WRITE); in rw_mgr_mem_calibrate_write_test()
1315 for (vg = seq->rwcfg->mem_virtual_groups_per_write_dqs - 1; in rw_mgr_mem_calibrate_write_test()
1321 seq->rwcfg->mem_virtual_groups_per_write_dqs in rw_mgr_mem_calibrate_write_test()
1323 rw_mgr_mem_calibrate_write_test_issue(seq, group, in rw_mgr_mem_calibrate_write_test()
1334 set_rank_and_odt_mask(seq, 0, RW_MGR_ODT_MODE_OFF); in rw_mgr_mem_calibrate_write_test()
1339 seq->param.write_correct_mask, in rw_mgr_mem_calibrate_write_test()
1340 *bit_chk == seq->param.write_correct_mask); in rw_mgr_mem_calibrate_write_test()
1341 return *bit_chk == seq->param.write_correct_mask; in rw_mgr_mem_calibrate_write_test()
1360 rw_mgr_mem_calibrate_read_test_patterns(struct socfpga_sdrseq *seq, in rw_mgr_mem_calibrate_read_test_patterns() argument
1367 (group * seq->rwcfg->mem_virtual_groups_per_read_dqs) in rw_mgr_mem_calibrate_read_test_patterns()
1370 seq->rwcfg->mem_number_of_ranks : in rw_mgr_mem_calibrate_read_test_patterns()
1372 const u32 shift_ratio = seq->rwcfg->mem_dq_per_read_dqs / in rw_mgr_mem_calibrate_read_test_patterns()
1373 seq->rwcfg->mem_virtual_groups_per_read_dqs; in rw_mgr_mem_calibrate_read_test_patterns()
1374 const u32 correct_mask_vg = seq->param.read_correct_mask_vg; in rw_mgr_mem_calibrate_read_test_patterns()
1380 bit_chk = seq->param.read_correct_mask; in rw_mgr_mem_calibrate_read_test_patterns()
1384 set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_READ_WRITE); in rw_mgr_mem_calibrate_read_test_patterns()
1388 writel(seq->rwcfg->guaranteed_read, in rw_mgr_mem_calibrate_read_test_patterns()
1392 writel(seq->rwcfg->guaranteed_read_cont, in rw_mgr_mem_calibrate_read_test_patterns()
1396 for (vg = seq->rwcfg->mem_virtual_groups_per_read_dqs - 1; in rw_mgr_mem_calibrate_read_test_patterns()
1402 writel(seq->rwcfg->guaranteed_read, in rw_mgr_mem_calibrate_read_test_patterns()
1413 writel(seq->rwcfg->clear_dqs_enable, addr + (group << 2)); in rw_mgr_mem_calibrate_read_test_patterns()
1415 set_rank_and_odt_mask(seq, 0, RW_MGR_ODT_MODE_OFF); in rw_mgr_mem_calibrate_read_test_patterns()
1417 if (bit_chk != seq->param.read_correct_mask) in rw_mgr_mem_calibrate_read_test_patterns()
1423 seq->param.read_correct_mask, ret); in rw_mgr_mem_calibrate_read_test_patterns()
1436 static void rw_mgr_mem_calibrate_read_load_patterns(struct socfpga_sdrseq *seq, in rw_mgr_mem_calibrate_read_load_patterns() argument
1441 seq->rwcfg->mem_number_of_ranks : in rw_mgr_mem_calibrate_read_load_patterns()
1449 set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_READ_WRITE); in rw_mgr_mem_calibrate_read_load_patterns()
1454 writel(seq->rwcfg->guaranteed_write_wait0, in rw_mgr_mem_calibrate_read_load_patterns()
1459 writel(seq->rwcfg->guaranteed_write_wait1, in rw_mgr_mem_calibrate_read_load_patterns()
1464 writel(seq->rwcfg->guaranteed_write_wait2, in rw_mgr_mem_calibrate_read_load_patterns()
1469 writel(seq->rwcfg->guaranteed_write_wait3, in rw_mgr_mem_calibrate_read_load_patterns()
1472 writel(seq->rwcfg->guaranteed_write, in rw_mgr_mem_calibrate_read_load_patterns()
1477 set_rank_and_odt_mask(seq, 0, RW_MGR_ODT_MODE_OFF); in rw_mgr_mem_calibrate_read_load_patterns()
1495 rw_mgr_mem_calibrate_read_test(struct socfpga_sdrseq *seq, in rw_mgr_mem_calibrate_read_test() argument
1501 const u32 rank_end = all_ranks ? seq->rwcfg->mem_number_of_ranks : in rw_mgr_mem_calibrate_read_test()
1505 seq->misccfg->enable_super_quick_calibration); in rw_mgr_mem_calibrate_read_test()
1506 u32 correct_mask_vg = seq->param.read_correct_mask_vg; in rw_mgr_mem_calibrate_read_test()
1513 *bit_chk = seq->param.read_correct_mask; in rw_mgr_mem_calibrate_read_test()
1517 set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_READ_WRITE); in rw_mgr_mem_calibrate_read_test()
1521 writel(seq->rwcfg->read_b2b_wait1, in rw_mgr_mem_calibrate_read_test()
1525 writel(seq->rwcfg->read_b2b_wait2, in rw_mgr_mem_calibrate_read_test()
1536 writel(seq->rwcfg->read_b2b, in rw_mgr_mem_calibrate_read_test()
1539 writel(seq->rwcfg->mem_if_read_dqs_width * in rw_mgr_mem_calibrate_read_test()
1540 seq->rwcfg->mem_virtual_groups_per_read_dqs - 1, in rw_mgr_mem_calibrate_read_test()
1545 writel(seq->rwcfg->read_b2b, in rw_mgr_mem_calibrate_read_test()
1549 for (vg = seq->rwcfg->mem_virtual_groups_per_read_dqs - 1; in rw_mgr_mem_calibrate_read_test()
1564 writel(seq->rwcfg->read_b2b, addr + in rw_mgr_mem_calibrate_read_test()
1566 seq->rwcfg->mem_virtual_groups_per_read_dqs + in rw_mgr_mem_calibrate_read_test()
1571 seq->rwcfg->mem_dq_per_read_dqs / in rw_mgr_mem_calibrate_read_test()
1572 seq->rwcfg->mem_virtual_groups_per_read_dqs; in rw_mgr_mem_calibrate_read_test()
1580 writel(seq->rwcfg->clear_dqs_enable, addr + (group << 2)); in rw_mgr_mem_calibrate_read_test()
1582 set_rank_and_odt_mask(seq, 0, RW_MGR_ODT_MODE_OFF); in rw_mgr_mem_calibrate_read_test()
1585 ret = (*bit_chk == seq->param.read_correct_mask); in rw_mgr_mem_calibrate_read_test()
1589 seq->param.read_correct_mask, ret); in rw_mgr_mem_calibrate_read_test()
1611 rw_mgr_mem_calibrate_read_test_all_ranks(struct socfpga_sdrseq *seq, in rw_mgr_mem_calibrate_read_test_all_ranks() argument
1617 return rw_mgr_mem_calibrate_read_test(seq, 0, grp, num_tries, in rw_mgr_mem_calibrate_read_test_all_ranks()
1639 static void rw_mgr_decr_vfifo(struct socfpga_sdrseq *seq, const u32 grp) in rw_mgr_decr_vfifo() argument
1643 for (i = 0; i < seq->misccfg->read_valid_fifo_size - 1; i++) in rw_mgr_decr_vfifo()
1653 static int find_vfifo_failing_read(struct socfpga_sdrseq *seq, in find_vfifo_failing_read() argument
1658 for (v = 0; v < seq->misccfg->read_valid_fifo_size; v++) { in find_vfifo_failing_read()
1661 ret = rw_mgr_mem_calibrate_read_test_all_ranks(seq, grp, 1, in find_vfifo_failing_read()
1690 static int sdr_find_phase_delay(struct socfpga_sdrseq *seq, int working, in sdr_find_phase_delay() argument
1694 const u32 max = delay ? seq->iocfg->dqs_en_delay_max : in sdr_find_phase_delay()
1695 seq->iocfg->dqs_en_phase_max; in sdr_find_phase_delay()
1700 scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, *pd); in sdr_find_phase_delay()
1702 scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, *pd); in sdr_find_phase_delay()
1704 ret = rw_mgr_mem_calibrate_read_test_all_ranks(seq, grp, 1, in sdr_find_phase_delay()
1728 static int sdr_find_phase(struct socfpga_sdrseq *seq, int working, in sdr_find_phase() argument
1731 const u32 end = seq->misccfg->read_valid_fifo_size + (working ? 0 : 1); in sdr_find_phase()
1738 ret = sdr_find_phase_delay(seq, working, 0, grp, work, in sdr_find_phase()
1739 seq->iocfg->delay_per_opa_tap, p); in sdr_find_phase()
1743 if (*p > seq->iocfg->dqs_en_phase_max) { in sdr_find_phase()
1764 static int sdr_working_phase(struct socfpga_sdrseq *seq, const u32 grp, in sdr_working_phase() argument
1767 const u32 dtaps_per_ptap = seq->iocfg->delay_per_opa_tap / in sdr_working_phase()
1768 seq->iocfg->delay_per_dqs_en_dchain_tap; in sdr_working_phase()
1775 scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, *d); in sdr_working_phase()
1776 ret = sdr_find_phase(seq, 1, grp, work_bgn, i, p); in sdr_working_phase()
1779 *work_bgn += seq->iocfg->delay_per_dqs_en_dchain_tap; in sdr_working_phase()
1796 static void sdr_backup_phase(struct socfpga_sdrseq *seq, const u32 grp, in sdr_backup_phase() argument
1804 *p = seq->iocfg->dqs_en_phase_max; in sdr_backup_phase()
1805 rw_mgr_decr_vfifo(seq, grp); in sdr_backup_phase()
1809 tmp_delay = *work_bgn - seq->iocfg->delay_per_opa_tap; in sdr_backup_phase()
1810 scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, *p); in sdr_backup_phase()
1812 for (d = 0; d <= seq->iocfg->dqs_en_delay_max && tmp_delay < *work_bgn; in sdr_backup_phase()
1814 scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, d); in sdr_backup_phase()
1816 ret = rw_mgr_mem_calibrate_read_test_all_ranks(seq, grp, 1, in sdr_backup_phase()
1823 tmp_delay += seq->iocfg->delay_per_dqs_en_dchain_tap; in sdr_backup_phase()
1828 if (*p > seq->iocfg->dqs_en_phase_max) { in sdr_backup_phase()
1833 scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, 0); in sdr_backup_phase()
1845 static int sdr_nonworking_phase(struct socfpga_sdrseq *seq, in sdr_nonworking_phase() argument
1851 *work_end += seq->iocfg->delay_per_opa_tap; in sdr_nonworking_phase()
1852 if (*p > seq->iocfg->dqs_en_phase_max) { in sdr_nonworking_phase()
1858 ret = sdr_find_phase(seq, 0, grp, work_end, i, p); in sdr_nonworking_phase()
1876 static int sdr_find_window_center(struct socfpga_sdrseq *seq, in sdr_find_window_center() argument
1889 tmp_delay = (seq->iocfg->dqs_en_phase_max + 1) in sdr_find_window_center()
1890 * seq->iocfg->delay_per_opa_tap; in sdr_find_window_center()
1896 tmp_delay = rounddown(work_mid, seq->iocfg->delay_per_opa_tap); in sdr_find_window_center()
1897 if (tmp_delay > seq->iocfg->dqs_en_phase_max in sdr_find_window_center()
1898 * seq->iocfg->delay_per_opa_tap) { in sdr_find_window_center()
1899 tmp_delay = seq->iocfg->dqs_en_phase_max in sdr_find_window_center()
1900 * seq->iocfg->delay_per_opa_tap; in sdr_find_window_center()
1902 p = tmp_delay / seq->iocfg->delay_per_opa_tap; in sdr_find_window_center()
1907 seq->iocfg->delay_per_dqs_en_dchain_tap); in sdr_find_window_center()
1908 if (d > seq->iocfg->dqs_en_delay_max) in sdr_find_window_center()
1909 d = seq->iocfg->dqs_en_delay_max; in sdr_find_window_center()
1910 tmp_delay += d * seq->iocfg->delay_per_dqs_en_dchain_tap; in sdr_find_window_center()
1914 scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, p); in sdr_find_window_center()
1915 scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, d); in sdr_find_window_center()
1921 for (i = 0; i < seq->misccfg->read_valid_fifo_size; i++) { in sdr_find_window_center()
1923 if (rw_mgr_mem_calibrate_read_test_all_ranks(seq, grp, 1, in sdr_find_window_center()
1949 rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(struct socfpga_sdrseq *seq, in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase() argument
1962 scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, 0); in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1963 scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, 0); in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1966 dtaps_per_ptap = seq->iocfg->delay_per_opa_tap / in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1967 seq->iocfg->delay_per_dqs_en_dchain_tap; in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1970 find_vfifo_failing_read(seq, grp); in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1974 ret = sdr_working_phase(seq, grp, &work_bgn, &d, &p, &i); in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1990 sdr_backup_phase(seq, grp, &work_bgn, &p); in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1996 ret = sdr_nonworking_phase(seq, grp, &work_end, &p, &i); in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
2004 p = seq->iocfg->dqs_en_phase_max; in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
2005 rw_mgr_decr_vfifo(seq, grp); in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
2010 work_end -= seq->iocfg->delay_per_opa_tap; in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
2011 scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, p); in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
2020 sdr_find_phase_delay(seq, 0, 1, grp, &work_end, in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
2021 seq->iocfg->delay_per_dqs_en_dchain_tap, &d); in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
2025 work_end -= seq->iocfg->delay_per_dqs_en_dchain_tap; in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
2051 p = seq->iocfg->dqs_en_phase_max; in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
2052 rw_mgr_decr_vfifo(seq, grp); in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
2061 scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, p); in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
2075 found_passing_read = !sdr_find_phase_delay(seq, 1, 1, grp, NULL, 0, &d); in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
2081 found_failing_read = !sdr_find_phase_delay(seq, 0, 1, grp, NULL, in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
2103 ret = sdr_find_window_center(seq, grp, work_bgn, work_end); in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
2121 static u32 search_stop_check(struct socfpga_sdrseq *seq, const int write, in search_stop_check() argument
2127 const u32 ratio = seq->rwcfg->mem_if_read_dqs_width / in search_stop_check()
2128 seq->rwcfg->mem_if_write_dqs_width; in search_stop_check()
2129 const u32 correct_mask = write ? seq->param.write_correct_mask : in search_stop_check()
2130 seq->param.read_correct_mask; in search_stop_check()
2131 const u32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs : in search_stop_check()
2132 seq->rwcfg->mem_dq_per_read_dqs; in search_stop_check()
2139 ret = !rw_mgr_mem_calibrate_write_test(seq, rank_bgn, in search_stop_check()
2144 ret = !rw_mgr_mem_calibrate_read_test(seq, rank_bgn, read_group, in search_stop_check()
2149 rw_mgr_mem_calibrate_write_test(seq, rank_bgn, write_group, 0, in search_stop_check()
2178 static void search_left_edge(struct socfpga_sdrseq *seq, const int write, in search_left_edge() argument
2184 const u32 delay_max = write ? seq->iocfg->io_out1_delay_max : in search_left_edge()
2185 seq->iocfg->io_in_delay_max; in search_left_edge()
2186 const u32 dqs_max = write ? seq->iocfg->io_out1_delay_max : in search_left_edge()
2187 seq->iocfg->dqs_in_delay_max; in search_left_edge()
2188 const u32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs : in search_left_edge()
2189 seq->rwcfg->mem_dq_per_read_dqs; in search_left_edge()
2195 scc_mgr_apply_group_dq_out1_delay(seq, d); in search_left_edge()
2197 scc_mgr_apply_group_dq_in_delay(seq, test_bgn, d); in search_left_edge()
2201 stop = search_stop_check(seq, write, d, rank_bgn, write_group, in search_left_edge()
2231 scc_mgr_apply_group_dq_out1_delay(seq, 0); in search_left_edge()
2233 scc_mgr_apply_group_dq_in_delay(seq, test_bgn, 0); in search_left_edge()
2289 static int search_right_edge(struct socfpga_sdrseq *seq, const int write, in search_right_edge() argument
2296 const u32 delay_max = write ? seq->iocfg->io_out1_delay_max : in search_right_edge()
2297 seq->iocfg->io_in_delay_max; in search_right_edge()
2298 const u32 dqs_max = write ? seq->iocfg->io_out1_delay_max : in search_right_edge()
2299 seq->iocfg->dqs_in_delay_max; in search_right_edge()
2300 const u32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs : in search_right_edge()
2301 seq->rwcfg->mem_dq_per_read_dqs; in search_right_edge()
2307 scc_mgr_apply_group_dqs_io_and_oct_out1(seq, in search_right_edge()
2312 if (seq->iocfg->shift_dqs_en_when_shift_dqs) { in search_right_edge()
2314 if (delay > seq->iocfg->dqs_en_delay_max) in search_right_edge()
2315 delay = seq->iocfg->dqs_en_delay_max; in search_right_edge()
2323 stop = search_stop_check(seq, write, d, rank_bgn, write_group, in search_right_edge()
2329 i < seq->rwcfg->mem_dq_per_write_dqs; in search_right_edge()
2417 static int get_window_mid_index(struct socfpga_sdrseq *seq, in get_window_mid_index() argument
2421 const u32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs : in get_window_mid_index()
2422 seq->rwcfg->mem_dq_per_read_dqs; in get_window_mid_index()
2465 static void center_dq_windows(struct socfpga_sdrseq *seq, in center_dq_windows() argument
2471 const s32 delay_max = write ? seq->iocfg->io_out1_delay_max : in center_dq_windows()
2472 seq->iocfg->io_in_delay_max; in center_dq_windows()
2473 const s32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs : in center_dq_windows()
2474 seq->rwcfg->mem_dq_per_read_dqs; in center_dq_windows()
2541 static int rw_mgr_mem_calibrate_vfifo_center(struct socfpga_sdrseq *seq, in rw_mgr_mem_calibrate_vfifo_center() argument
2556 s32 left_edge[seq->rwcfg->mem_dq_per_read_dqs]; in rw_mgr_mem_calibrate_vfifo_center()
2557 s32 right_edge[seq->rwcfg->mem_dq_per_read_dqs]; in rw_mgr_mem_calibrate_vfifo_center()
2567 if (seq->iocfg->shift_dqs_en_when_shift_dqs) in rw_mgr_mem_calibrate_vfifo_center()
2568 start_dqs_en = readl(addr - seq->iocfg->dqs_en_delay_offset); in rw_mgr_mem_calibrate_vfifo_center()
2573 for (i = 0; i < seq->rwcfg->mem_dq_per_read_dqs; i++) { in rw_mgr_mem_calibrate_vfifo_center()
2574 left_edge[i] = seq->iocfg->io_in_delay_max + 1; in rw_mgr_mem_calibrate_vfifo_center()
2575 right_edge[i] = seq->iocfg->io_in_delay_max + 1; in rw_mgr_mem_calibrate_vfifo_center()
2579 search_left_edge(seq, 0, rank_bgn, rw_group, rw_group, test_bgn, in rw_mgr_mem_calibrate_vfifo_center()
2585 ret = search_right_edge(seq, 0, rank_bgn, rw_group, rw_group, in rw_mgr_mem_calibrate_vfifo_center()
2596 if (seq->iocfg->shift_dqs_en_when_shift_dqs) in rw_mgr_mem_calibrate_vfifo_center()
2606 set_failing_group_stage(seq, rw_group * in rw_mgr_mem_calibrate_vfifo_center()
2607 seq->rwcfg->mem_dq_per_read_dqs + i, in rw_mgr_mem_calibrate_vfifo_center()
2611 set_failing_group_stage(seq, rw_group * in rw_mgr_mem_calibrate_vfifo_center()
2612 seq->rwcfg->mem_dq_per_read_dqs + i, in rw_mgr_mem_calibrate_vfifo_center()
2619 min_index = get_window_mid_index(seq, 0, left_edge, right_edge, in rw_mgr_mem_calibrate_vfifo_center()
2625 if (new_dqs > seq->iocfg->dqs_in_delay_max) in rw_mgr_mem_calibrate_vfifo_center()
2626 new_dqs = seq->iocfg->dqs_in_delay_max; in rw_mgr_mem_calibrate_vfifo_center()
2634 if (seq->iocfg->shift_dqs_en_when_shift_dqs) { in rw_mgr_mem_calibrate_vfifo_center()
2635 if (start_dqs_en - mid_min > seq->iocfg->dqs_en_delay_max) in rw_mgr_mem_calibrate_vfifo_center()
2637 seq->iocfg->dqs_en_delay_max; in rw_mgr_mem_calibrate_vfifo_center()
2646 seq->iocfg->shift_dqs_en_when_shift_dqs ? start_dqs_en : -1, in rw_mgr_mem_calibrate_vfifo_center()
2650 center_dq_windows(seq, 0, left_edge, right_edge, mid_min, orig_mid_min, in rw_mgr_mem_calibrate_vfifo_center()
2654 if (seq->iocfg->shift_dqs_en_when_shift_dqs) { in rw_mgr_mem_calibrate_vfifo_center()
2689 static int rw_mgr_mem_calibrate_guaranteed_write(struct socfpga_sdrseq *seq, in rw_mgr_mem_calibrate_guaranteed_write() argument
2696 scc_mgr_set_dqdqs_output_phase_all_ranks(seq, rw_group, phase); in rw_mgr_mem_calibrate_guaranteed_write()
2706 rw_mgr_mem_calibrate_read_load_patterns(seq, 0, 1); in rw_mgr_mem_calibrate_guaranteed_write()
2708 if (seq->gbl.phy_debug_mode_flags & PHY_DEBUG_DISABLE_GUARANTEED_READ) in rw_mgr_mem_calibrate_guaranteed_write()
2715 ret = rw_mgr_mem_calibrate_read_test_patterns(seq, 0, rw_group, 1); in rw_mgr_mem_calibrate_guaranteed_write()
2732 rw_mgr_mem_calibrate_dqs_enable_calibration(struct socfpga_sdrseq *seq, in rw_mgr_mem_calibrate_dqs_enable_calibration() argument
2742 const u32 delay_step = seq->iocfg->io_in_delay_max / in rw_mgr_mem_calibrate_dqs_enable_calibration()
2743 (seq->rwcfg->mem_dq_per_read_dqs - 1); in rw_mgr_mem_calibrate_dqs_enable_calibration()
2750 for (r = 0; r < seq->rwcfg->mem_number_of_ranks; in rw_mgr_mem_calibrate_dqs_enable_calibration()
2753 i < seq->rwcfg->mem_dq_per_read_dqs; in rw_mgr_mem_calibrate_dqs_enable_calibration()
2770 ret = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(seq, rw_group); in rw_mgr_mem_calibrate_dqs_enable_calibration()
2776 for (r = 0; r < seq->rwcfg->mem_number_of_ranks; in rw_mgr_mem_calibrate_dqs_enable_calibration()
2778 scc_mgr_apply_group_dq_in_delay(seq, test_bgn, 0); in rw_mgr_mem_calibrate_dqs_enable_calibration()
2796 rw_mgr_mem_calibrate_dq_dqs_centering(struct socfpga_sdrseq *seq, in rw_mgr_mem_calibrate_dq_dqs_centering() argument
2811 rank_bgn < seq->rwcfg->mem_number_of_ranks; in rw_mgr_mem_calibrate_dq_dqs_centering()
2813 ret = rw_mgr_mem_calibrate_vfifo_center(seq, rank_bgn, rw_group, in rw_mgr_mem_calibrate_dq_dqs_centering()
2844 static int rw_mgr_mem_calibrate_vfifo(struct socfpga_sdrseq *seq, in rw_mgr_mem_calibrate_vfifo() argument
2863 dtaps_per_ptap = DIV_ROUND_UP(seq->iocfg->delay_per_opa_tap, in rw_mgr_mem_calibrate_vfifo()
2864 seq->iocfg->delay_per_dqs_en_dchain_tap) in rw_mgr_mem_calibrate_vfifo()
2875 scc_mgr_apply_group_all_out_delay_add_all_ranks(seq, in rw_mgr_mem_calibrate_vfifo()
2880 for (p = 0; p <= seq->iocfg->dqdqs_out_phase_max; p++) { in rw_mgr_mem_calibrate_vfifo()
2882 ret = rw_mgr_mem_calibrate_guaranteed_write(seq, in rw_mgr_mem_calibrate_vfifo()
2889 ret = rw_mgr_mem_calibrate_dqs_enable_calibration(seq, in rw_mgr_mem_calibrate_vfifo()
2902 ret = rw_mgr_mem_calibrate_dq_dqs_centering(seq, in rw_mgr_mem_calibrate_vfifo()
2917 set_failing_group_stage(seq, rw_group, CAL_STAGE_VFIFO, in rw_mgr_mem_calibrate_vfifo()
2929 scc_mgr_zero_group(seq, rw_group, 1); in rw_mgr_mem_calibrate_vfifo()
2944 static int rw_mgr_mem_calibrate_vfifo_end(struct socfpga_sdrseq *seq, in rw_mgr_mem_calibrate_vfifo_end() argument
2957 ret = rw_mgr_mem_calibrate_dq_dqs_centering(seq, rw_group, test_bgn, 0, in rw_mgr_mem_calibrate_vfifo_end()
2960 set_failing_group_stage(seq, rw_group, in rw_mgr_mem_calibrate_vfifo_end()
2975 static u32 rw_mgr_mem_calibrate_lfifo(struct socfpga_sdrseq *seq) in rw_mgr_mem_calibrate_lfifo() argument
2986 rw_mgr_mem_calibrate_read_load_patterns(seq, 0, 1); in rw_mgr_mem_calibrate_lfifo()
2989 writel(seq->gbl.curr_read_lat, &phy_mgr_cfg->phy_rlat); in rw_mgr_mem_calibrate_lfifo()
2991 __func__, __LINE__, seq->gbl.curr_read_lat); in rw_mgr_mem_calibrate_lfifo()
2993 if (!rw_mgr_mem_calibrate_read_test_all_ranks(seq, 0, in rw_mgr_mem_calibrate_lfifo()
3003 seq->gbl.curr_read_lat--; in rw_mgr_mem_calibrate_lfifo()
3004 } while (seq->gbl.curr_read_lat > 0); in rw_mgr_mem_calibrate_lfifo()
3011 seq->gbl.curr_read_lat += 2; in rw_mgr_mem_calibrate_lfifo()
3012 writel(seq->gbl.curr_read_lat, &phy_mgr_cfg->phy_rlat); in rw_mgr_mem_calibrate_lfifo()
3015 __func__, __LINE__, seq->gbl.curr_read_lat); in rw_mgr_mem_calibrate_lfifo()
3017 set_failing_group_stage(seq, 0xff, CAL_STAGE_LFIFO, in rw_mgr_mem_calibrate_lfifo()
3022 __func__, __LINE__, seq->gbl.curr_read_lat); in rw_mgr_mem_calibrate_lfifo()
3043 static void search_window(struct socfpga_sdrseq *seq, in search_window() argument
3050 const int max = seq->iocfg->io_out1_delay_max - new_dqs; in search_window()
3057 scc_mgr_apply_group_dm_out1_delay(seq, d); in search_window()
3065 scc_mgr_apply_group_dqs_io_and_oct_out1(seq, in search_window()
3072 if (rw_mgr_mem_calibrate_write_test(seq, rank_bgn, write_group, in search_window()
3082 if (*bgn_curr == seq->iocfg->io_out1_delay_max + 1) in search_window()
3096 *bgn_curr = seq->iocfg->io_out1_delay_max + 1; in search_window()
3097 *end_curr = seq->iocfg->io_out1_delay_max + 1; in search_window()
3108 if (*win_best - 1 > seq->iocfg->io_out1_delay_max in search_window()
3125 rw_mgr_mem_calibrate_writes_center(struct socfpga_sdrseq *seq, in rw_mgr_mem_calibrate_writes_center() argument
3132 int left_edge[seq->rwcfg->mem_dq_per_write_dqs]; in rw_mgr_mem_calibrate_writes_center()
3133 int right_edge[seq->rwcfg->mem_dq_per_write_dqs]; in rw_mgr_mem_calibrate_writes_center()
3138 int bgn_curr = seq->iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
3139 int end_curr = seq->iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
3140 int bgn_best = seq->iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
3141 int end_best = seq->iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
3152 (seq->rwcfg->mem_dq_per_write_dqs << 2)); in rw_mgr_mem_calibrate_writes_center()
3161 for (i = 0; i < seq->rwcfg->mem_dq_per_write_dqs; i++) { in rw_mgr_mem_calibrate_writes_center()
3162 left_edge[i] = seq->iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
3163 right_edge[i] = seq->iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
3167 search_left_edge(seq, 1, rank_bgn, write_group, 0, test_bgn, in rw_mgr_mem_calibrate_writes_center()
3172 ret = search_right_edge(seq, 1, rank_bgn, write_group, 0, in rw_mgr_mem_calibrate_writes_center()
3177 set_failing_group_stage(seq, test_bgn + ret - 1, in rw_mgr_mem_calibrate_writes_center()
3183 min_index = get_window_mid_index(seq, 1, left_edge, right_edge, in rw_mgr_mem_calibrate_writes_center()
3195 center_dq_windows(seq, 1, left_edge, right_edge, mid_min, orig_mid_min, in rw_mgr_mem_calibrate_writes_center()
3199 scc_mgr_apply_group_dqs_io_and_oct_out1(seq, write_group, new_dqs); in rw_mgr_mem_calibrate_writes_center()
3206 search_window(seq, 1, rank_bgn, write_group, &bgn_curr, &end_curr, in rw_mgr_mem_calibrate_writes_center()
3210 scc_mgr_apply_group_dm_out1_delay(seq, 0); in rw_mgr_mem_calibrate_writes_center()
3218 bgn_curr = seq->iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
3219 end_curr = seq->iocfg->io_out1_delay_max + 1; in rw_mgr_mem_calibrate_writes_center()
3223 search_window(seq, 0, rank_bgn, write_group, &bgn_curr, &end_curr, in rw_mgr_mem_calibrate_writes_center()
3234 scc_mgr_apply_group_dqs_io_and_oct_out1(seq, write_group, new_dqs); in rw_mgr_mem_calibrate_writes_center()
3251 scc_mgr_apply_group_dm_out1_delay(seq, mid); in rw_mgr_mem_calibrate_writes_center()
3259 seq->gbl.fom_out += dq_margin + dqs_margin; in rw_mgr_mem_calibrate_writes_center()
3288 static int rw_mgr_mem_calibrate_writes(struct socfpga_sdrseq *seq, in rw_mgr_mem_calibrate_writes() argument
3301 ret = rw_mgr_mem_calibrate_writes_center(seq, rank_bgn, group, in rw_mgr_mem_calibrate_writes()
3304 set_failing_group_stage(seq, group, CAL_STAGE_WRITES, in rw_mgr_mem_calibrate_writes()
3315 static void mem_precharge_and_activate(struct socfpga_sdrseq *seq) in mem_precharge_and_activate() argument
3319 for (r = 0; r < seq->rwcfg->mem_number_of_ranks; r++) { in mem_precharge_and_activate()
3321 set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_OFF); in mem_precharge_and_activate()
3324 writel(seq->rwcfg->precharge_all, SDR_PHYGRP_RWMGRGRP_ADDRESS | in mem_precharge_and_activate()
3328 writel(seq->rwcfg->activate_0_and_1_wait1, in mem_precharge_and_activate()
3332 writel(seq->rwcfg->activate_0_and_1_wait2, in mem_precharge_and_activate()
3336 writel(seq->rwcfg->activate_0_and_1, in mem_precharge_and_activate()
3347 static void mem_init_latency(struct socfpga_sdrseq *seq) in mem_init_latency() argument
3354 const u32 max_latency = (1 << seq->misccfg->max_latency_count_width) in mem_init_latency()
3367 seq->gbl.rw_wl_nop_cycles = wlat - 1; in mem_init_latency()
3373 seq->gbl.curr_read_lat = rlat + 16; in mem_init_latency()
3374 if (seq->gbl.curr_read_lat > max_latency) in mem_init_latency()
3375 seq->gbl.curr_read_lat = max_latency; in mem_init_latency()
3377 writel(seq->gbl.curr_read_lat, &phy_mgr_cfg->phy_rlat); in mem_init_latency()
3388 static void mem_skip_calibrate(struct socfpga_sdrseq *seq) in mem_skip_calibrate() argument
3395 for (r = 0; r < seq->rwcfg->mem_number_of_ranks; in mem_skip_calibrate()
3401 for (i = 0; i < seq->rwcfg->mem_if_read_dqs_width; i++) { in mem_skip_calibrate()
3403 if (seq->iocfg->dll_chain_length == 6) in mem_skip_calibrate()
3435 ((125 * seq->iocfg->dll_chain_length) in mem_skip_calibrate()
3441 for (i = 0; i < seq->rwcfg->mem_if_write_dqs_width; i++) { in mem_skip_calibrate()
3451 for (i = 0; i < seq->rwcfg->mem_if_read_dqs_width; i++) { in mem_skip_calibrate()
3461 vfifo_offset = seq->misccfg->calib_vfifo_offset; in mem_skip_calibrate()
3470 seq->gbl.curr_read_lat = seq->misccfg->calib_lfifo_offset; in mem_skip_calibrate()
3471 writel(seq->gbl.curr_read_lat, &phy_mgr_cfg->phy_rlat); in mem_skip_calibrate()
3479 static u32 mem_calibrate(struct socfpga_sdrseq *seq) in mem_calibrate() argument
3489 const u32 rwdqs_ratio = seq->rwcfg->mem_if_read_dqs_width / in mem_calibrate()
3490 seq->rwcfg->mem_if_write_dqs_width; in mem_calibrate()
3495 seq->gbl.error_substage = CAL_SUBSTAGE_NIL; in mem_calibrate()
3496 seq->gbl.error_stage = CAL_STAGE_NIL; in mem_calibrate()
3497 seq->gbl.error_group = 0xff; in mem_calibrate()
3498 seq->gbl.fom_in = 0; in mem_calibrate()
3499 seq->gbl.fom_out = 0; in mem_calibrate()
3502 mem_init_latency(seq); in mem_calibrate()
3505 mem_precharge_and_activate(seq); in mem_calibrate()
3507 for (i = 0; i < seq->rwcfg->mem_if_read_dqs_width; i++) { in mem_calibrate()
3518 if ((seq->dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) { in mem_calibrate()
3523 mem_skip_calibrate(seq); in mem_calibrate()
3539 scc_mgr_zero_all(seq); in mem_calibrate()
3544 < seq->rwcfg->mem_if_write_dqs_width; write_group++, in mem_calibrate()
3545 write_test_bgn += seq->rwcfg->mem_dq_per_write_dqs) { in mem_calibrate()
3559 scc_mgr_zero_group(seq, write_group, 0); in mem_calibrate()
3565 read_test_bgn += seq->rwcfg->mem_dq_per_read_dqs) { in mem_calibrate()
3570 if (rw_mgr_mem_calibrate_vfifo(seq, read_group, in mem_calibrate()
3574 if (!(seq->gbl.phy_debug_mode_flags & in mem_calibrate()
3584 rank_bgn < seq->rwcfg->mem_number_of_ranks; in mem_calibrate()
3595 if (!rw_mgr_mem_calibrate_writes(seq, rank_bgn, in mem_calibrate()
3601 if (!(seq->gbl.phy_debug_mode_flags & in mem_calibrate()
3614 read_test_bgn += seq->rwcfg->mem_dq_per_read_dqs) { in mem_calibrate()
3618 if (!rw_mgr_mem_calibrate_vfifo_end(seq, in mem_calibrate()
3623 if (!(seq->gbl.phy_debug_mode_flags & in mem_calibrate()
3649 if (!rw_mgr_mem_calibrate_lfifo(seq)) in mem_calibrate()
3666 static int run_mem_calibrate(struct socfpga_sdrseq *seq) in run_mem_calibrate() argument
3681 phy_mgr_initialize(seq); in run_mem_calibrate()
3682 rw_mgr_mem_initialize(seq); in run_mem_calibrate()
3685 pass = mem_calibrate(seq); in run_mem_calibrate()
3687 mem_precharge_and_activate(seq); in run_mem_calibrate()
3691 rw_mgr_mem_handoff(seq); in run_mem_calibrate()
3712 static void debug_mem_calibrate(struct socfpga_sdrseq *seq, int pass) in debug_mem_calibrate() argument
3719 seq->gbl.fom_in /= 2; in debug_mem_calibrate()
3720 seq->gbl.fom_out /= 2; in debug_mem_calibrate()
3722 if (seq->gbl.fom_in > 0xff) in debug_mem_calibrate()
3723 seq->gbl.fom_in = 0xff; in debug_mem_calibrate()
3725 if (seq->gbl.fom_out > 0xff) in debug_mem_calibrate()
3726 seq->gbl.fom_out = 0xff; in debug_mem_calibrate()
3729 debug_info = seq->gbl.fom_in; in debug_mem_calibrate()
3730 debug_info |= seq->gbl.fom_out << 8; in debug_mem_calibrate()
3738 debug_info = seq->gbl.error_stage; in debug_mem_calibrate()
3739 debug_info |= seq->gbl.error_substage << 8; in debug_mem_calibrate()
3740 debug_info |= seq->gbl.error_group << 16; in debug_mem_calibrate()
3747 debug_info = seq->gbl.error_stage; in debug_mem_calibrate()
3748 debug_info |= seq->gbl.error_substage << 8; in debug_mem_calibrate()
3749 debug_info |= seq->gbl.error_group << 16; in debug_mem_calibrate()
3783 static void initialize_reg_file(struct socfpga_sdrseq *seq) in initialize_reg_file() argument
3786 writel(seq->misccfg->reg_file_init_seq_signature, in initialize_reg_file()
3851 static void initialize_tracking(struct socfpga_sdrseq *seq) in initialize_tracking() argument
3858 writel(DIV_ROUND_UP(seq->iocfg->delay_per_opa_tap, in initialize_tracking()
3859 seq->iocfg->delay_per_dchain_tap) - 1, in initialize_tracking()
3881 writel((seq->rwcfg->idle << 24) | in initialize_tracking()
3882 (seq->rwcfg->activate_1 << 16) | in initialize_tracking()
3883 (seq->rwcfg->sgle_read << 8) | in initialize_tracking()
3884 (seq->rwcfg->precharge_all << 0), in initialize_tracking()
3888 writel(seq->rwcfg->mem_if_read_dqs_width, in initialize_tracking()
3895 writel((seq->rwcfg->refresh_all << 24) | (1000 << 0), in initialize_tracking()
3903 struct socfpga_sdrseq seq; in sdram_calibration_full() local
3912 memset(&seq, 0, sizeof(seq)); in sdram_calibration_full()
3914 seq.rwcfg = socfpga_get_sdram_rwmgr_config(); in sdram_calibration_full()
3915 seq.iocfg = socfpga_get_sdram_io_config(); in sdram_calibration_full()
3916 seq.misccfg = socfpga_get_sdram_misc_config(); in sdram_calibration_full()
3919 seq.gbl.phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT; in sdram_calibration_full()
3925 seq.gbl.phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ; in sdram_calibration_full()
3928 initialize_reg_file(&seq); in sdram_calibration_full()
3935 initialize_tracking(&seq); in sdram_calibration_full()
3942 seq.rwcfg->mem_number_of_ranks, in sdram_calibration_full()
3943 seq.rwcfg->mem_number_of_cs_per_dimm, in sdram_calibration_full()
3944 seq.rwcfg->mem_dq_per_read_dqs, in sdram_calibration_full()
3945 seq.rwcfg->mem_dq_per_write_dqs, in sdram_calibration_full()
3946 seq.rwcfg->mem_virtual_groups_per_read_dqs, in sdram_calibration_full()
3947 seq.rwcfg->mem_virtual_groups_per_write_dqs); in sdram_calibration_full()
3950 seq.rwcfg->mem_if_read_dqs_width, in sdram_calibration_full()
3951 seq.rwcfg->mem_if_write_dqs_width, in sdram_calibration_full()
3952 seq.rwcfg->mem_data_width, seq.rwcfg->mem_data_mask_width, in sdram_calibration_full()
3953 seq.iocfg->delay_per_opa_tap, in sdram_calibration_full()
3954 seq.iocfg->delay_per_dchain_tap); in sdram_calibration_full()
3956 seq.iocfg->delay_per_dqs_en_dchain_tap, in sdram_calibration_full()
3957 seq.iocfg->dll_chain_length); in sdram_calibration_full()
3960 seq.iocfg->dqs_en_phase_max, seq.iocfg->dqdqs_out_phase_max, in sdram_calibration_full()
3961 seq.iocfg->dqs_en_delay_max, seq.iocfg->dqs_in_delay_max); in sdram_calibration_full()
3963 seq.iocfg->io_in_delay_max, seq.iocfg->io_out1_delay_max, in sdram_calibration_full()
3964 seq.iocfg->io_out2_delay_max); in sdram_calibration_full()
3966 seq.iocfg->dqs_in_reserve, seq.iocfg->dqs_out_reserve); in sdram_calibration_full()
3978 seq.dyn_calib_steps = STATIC_CALIB_STEPS; in sdram_calibration_full()
3983 if (!(seq.dyn_calib_steps & CALIB_SKIP_DELAY_LOOPS)) in sdram_calibration_full()
3984 seq.skip_delay_mask = 0xff; in sdram_calibration_full()
3986 seq.skip_delay_mask = 0x0; in sdram_calibration_full()
3988 pass = run_mem_calibrate(&seq); in sdram_calibration_full()
3989 debug_mem_calibrate(&seq, pass); in sdram_calibration_full()