]> git.ipfire.org Git - thirdparty/linux.git/blame - drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
projects / thirdparty / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'drm/tegra/for-5.7-fixes' of git://anongit.freedesktop.org/tegra/linux...
[thirdparty/linux.git] / drivers / net / ethernet / stmicro / stmmac / stmmac_main.c
CommitLineData
4fa9c49f 1// SPDX-License-Identifier: GPL-2.0-only
47dd7a54
GC		 2/*******************************************************************************
3 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
4 ST Ethernet IPs are built around a Synopsys IP Core.
5
286a8372 6 Copyright(C) 2007-2011 STMicroelectronics Ltd
47dd7a54 7
47dd7a54
GC		 8
9 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
10
11 Documentation available at:
12 http://www.stlinux.com
13 Support available at:
14 https://bugzilla.stlinux.com/
15*******************************************************************************/
16
6a81c26f 17#include <linux/clk.h>
47dd7a54
GC		 18#include <linux/kernel.h>
19#include <linux/interrupt.h>
47dd7a54
GC		 20#include <linux/ip.h>
21#include <linux/tcp.h>
22#include <linux/skbuff.h>
23#include <linux/ethtool.h>
24#include <linux/if_ether.h>
25#include <linux/crc32.h>
26#include <linux/mii.h>
01789349 27#include <linux/if.h>
47dd7a54
GC		 28#include <linux/if_vlan.h>
29#include <linux/dma-mapping.h>
5a0e3ad6 30#include <linux/slab.h>
70c71606 31#include <linux/prefetch.h>
db88f10a 32#include <linux/pinctrl/consumer.h>
50fb4f74 33#ifdef CONFIG_DEBUG_FS
7ac29055
GC		 34#include <linux/debugfs.h>
35#include <linux/seq_file.h>
50fb4f74 36#endif /* CONFIG_DEBUG_FS */
891434b1 37#include <linux/net_tstamp.h>
eeef2f6b 38#include <linux/phylink.h>
b7766206 39#include <linux/udp.h>
4dbbe8dd 40#include <net/pkt_cls.h>
891434b1 41#include "stmmac_ptp.h"
286a8372 42#include "stmmac.h"
c5e4ddbd 43#include <linux/reset.h>
5790cf3c 44#include <linux/of_mdio.h>
19d857c9 45#include "dwmac1000.h"
7d9e6c5a 46#include "dwxgmac2.h"
42de047d 47#include "hwif.h"
47dd7a54 48
8d558f02 49#define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16)
f748be53 50#define TSO_MAX_BUFF_SIZE (SZ_16K - 1)
47dd7a54
GC		 51
52/* Module parameters */
32ceabca 53#define TX_TIMEO 5000
47dd7a54 54static int watchdog = TX_TIMEO;
d3757ba4 55module_param(watchdog, int, 0644);
32ceabca 56MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
47dd7a54 57
32ceabca 58static int debug = -1;
d3757ba4 59module_param(debug, int, 0644);
32ceabca 60MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
47dd7a54 61
47d1f71f 62static int phyaddr = -1;
d3757ba4 63module_param(phyaddr, int, 0444);
47dd7a54
GC		 64MODULE_PARM_DESC(phyaddr, "Physical device address");
65
e3ad57c9 66#define STMMAC_TX_THRESH (DMA_TX_SIZE / 4)
120e87f9 67#define STMMAC_RX_THRESH (DMA_RX_SIZE / 4)
47dd7a54 68
e9989339 69static int flow_ctrl = FLOW_AUTO;
d3757ba4 70module_param(flow_ctrl, int, 0644);
47dd7a54
GC		 71MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");
72
73static int pause = PAUSE_TIME;
d3757ba4 74module_param(pause, int, 0644);
47dd7a54
GC		 75MODULE_PARM_DESC(pause, "Flow Control Pause Time");
76
77#define TC_DEFAULT 64
78static int tc = TC_DEFAULT;
d3757ba4 79module_param(tc, int, 0644);
47dd7a54
GC		 80MODULE_PARM_DESC(tc, "DMA threshold control value");
81
d916701c
GC		 82#define DEFAULT_BUFSIZE 1536
83static int buf_sz = DEFAULT_BUFSIZE;
d3757ba4 84module_param(buf_sz, int, 0644);
47dd7a54
GC		 85MODULE_PARM_DESC(buf_sz, "DMA buffer size");
86
22ad3838
GC		 87#define STMMAC_RX_COPYBREAK 256
88
47dd7a54
GC		 89static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
90 NETIF_MSG_LINK | NETIF_MSG_IFUP |
91 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
92
d765955d
GC		 93#define STMMAC_DEFAULT_LPI_TIMER 1000
94static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
d3757ba4 95module_param(eee_timer, int, 0644);
d765955d 96MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
f5351ef7 97#define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x))
d765955d 98
22d3efe5
PM		 99/* By default the driver will use the ring mode to manage tx and rx descriptors,
100 * but allow user to force to use the chain instead of the ring
4a7d666a
GC		 101 */
102static unsigned int chain_mode;
d3757ba4 103module_param(chain_mode, int, 0444);
4a7d666a
GC		 104MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
105
47dd7a54 106static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
47dd7a54 107
50fb4f74 108#ifdef CONFIG_DEBUG_FS
481a7d15 109static const struct net_device_ops stmmac_netdev_ops;
8d72ab11 110static void stmmac_init_fs(struct net_device *dev);
466c5ac8 111static void stmmac_exit_fs(struct net_device *dev);
bfab27a1
GC		 112#endif
113
9125cdd1
GC		 114#define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))
115
47dd7a54
GC		 116/**
117 * stmmac_verify_args - verify the driver parameters.
732fdf0e
GC		 118 * Description: it checks the driver parameters and set a default in case of
119 * errors.
47dd7a54
GC		 120 */
121static void stmmac_verify_args(void)
122{
123 if (unlikely(watchdog < 0))
124 watchdog = TX_TIMEO;
d916701c
GC		 125 if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
126 buf_sz = DEFAULT_BUFSIZE;
47dd7a54
GC		 127 if (unlikely(flow_ctrl > 1))
128 flow_ctrl = FLOW_AUTO;
129 else if (likely(flow_ctrl < 0))
130 flow_ctrl = FLOW_OFF;
131 if (unlikely((pause < 0) || (pause > 0xffff)))
132 pause = PAUSE_TIME;
d765955d
GC		 133 if (eee_timer < 0)
134 eee_timer = STMMAC_DEFAULT_LPI_TIMER;
47dd7a54
GC		 135}
136
c22a3f48
JP		 137/**
138 * stmmac_disable_all_queues - Disable all queues
139 * @priv: driver private structure
140 */
141static void stmmac_disable_all_queues(struct stmmac_priv *priv)
142{
143 u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
8fce3331
JA		 144 u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
145 u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
c22a3f48
JP		 146 u32 queue;
147
8fce3331
JA		 148 for (queue = 0; queue < maxq; queue++) {
149 struct stmmac_channel *ch = &priv->channel[queue];
c22a3f48 150
4ccb4585
JA		 151 if (queue < rx_queues_cnt)
152 napi_disable(&ch->rx_napi);
153 if (queue < tx_queues_cnt)
154 napi_disable(&ch->tx_napi);
c22a3f48
JP		 155 }
156}
157
158/**
159 * stmmac_enable_all_queues - Enable all queues
160 * @priv: driver private structure
161 */
162static void stmmac_enable_all_queues(struct stmmac_priv *priv)
163{
164 u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
8fce3331
JA		 165 u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
166 u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
c22a3f48
JP		 167 u32 queue;
168
8fce3331
JA		 169 for (queue = 0; queue < maxq; queue++) {
170 struct stmmac_channel *ch = &priv->channel[queue];
c22a3f48 171
4ccb4585
JA		 172 if (queue < rx_queues_cnt)
173 napi_enable(&ch->rx_napi);
174 if (queue < tx_queues_cnt)
175 napi_enable(&ch->tx_napi);
c22a3f48
JP		 176 }
177}
178
179/**
180 * stmmac_stop_all_queues - Stop all queues
181 * @priv: driver private structure
182 */
183static void stmmac_stop_all_queues(struct stmmac_priv *priv)
184{
185 u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
186 u32 queue;
187
188 for (queue = 0; queue < tx_queues_cnt; queue++)
189 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
190}
191
192/**
193 * stmmac_start_all_queues - Start all queues
194 * @priv: driver private structure
195 */
196static void stmmac_start_all_queues(struct stmmac_priv *priv)
197{
198 u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
199 u32 queue;
200
201 for (queue = 0; queue < tx_queues_cnt; queue++)
202 netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue));
203}
204
34877a15
JA		 205static void stmmac_service_event_schedule(struct stmmac_priv *priv)
206{
207 if (!test_bit(STMMAC_DOWN, &priv->state) &&
208 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state))
209 queue_work(priv->wq, &priv->service_task);
210}
211
212static void stmmac_global_err(struct stmmac_priv *priv)
213{
214 netif_carrier_off(priv->dev);
215 set_bit(STMMAC_RESET_REQUESTED, &priv->state);
216 stmmac_service_event_schedule(priv);
217}
218
32ceabca
GC		 219/**
220 * stmmac_clk_csr_set - dynamically set the MDC clock
221 * @priv: driver private structure
222 * Description: this is to dynamically set the MDC clock according to the csr
223 * clock input.
224 * Note:
225 * If a specific clk_csr value is passed from the platform
226 * this means that the CSR Clock Range selection cannot be
227 * changed at run-time and it is fixed (as reported in the driver
228 * documentation). Viceversa the driver will try to set the MDC
229 * clock dynamically according to the actual clock input.
230 */
cd7201f4
GC		 231static void stmmac_clk_csr_set(struct stmmac_priv *priv)
232{
cd7201f4
GC		 233 u32 clk_rate;
234
f573c0b9 235 clk_rate = clk_get_rate(priv->plat->stmmac_clk);
cd7201f4
GC		 236
237 /* Platform provided default clk_csr would be assumed valid
ceb69499
GC		 238 * for all other cases except for the below mentioned ones.
239 * For values higher than the IEEE 802.3 specified frequency
240 * we can not estimate the proper divider as it is not known
241 * the frequency of clk_csr_i. So we do not change the default
242 * divider.
243 */
cd7201f4
GC		 244 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
245 if (clk_rate < CSR_F_35M)
246 priv->clk_csr = STMMAC_CSR_20_35M;
247 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
248 priv->clk_csr = STMMAC_CSR_35_60M;
249 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
250 priv->clk_csr = STMMAC_CSR_60_100M;
251 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
252 priv->clk_csr = STMMAC_CSR_100_150M;
253 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
254 priv->clk_csr = STMMAC_CSR_150_250M;
19d857c9 255 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M))
cd7201f4 256 priv->clk_csr = STMMAC_CSR_250_300M;
ceb69499 257 }
9f93ac8d
LC		 258
259 if (priv->plat->has_sun8i) {
260 if (clk_rate > 160000000)
261 priv->clk_csr = 0x03;
262 else if (clk_rate > 80000000)
263 priv->clk_csr = 0x02;
264 else if (clk_rate > 40000000)
265 priv->clk_csr = 0x01;
266 else
267 priv->clk_csr = 0;
268 }
7d9e6c5a
JA		 269
270 if (priv->plat->has_xgmac) {
271 if (clk_rate > 400000000)
272 priv->clk_csr = 0x5;
273 else if (clk_rate > 350000000)
274 priv->clk_csr = 0x4;
275 else if (clk_rate > 300000000)
276 priv->clk_csr = 0x3;
277 else if (clk_rate > 250000000)
278 priv->clk_csr = 0x2;
279 else if (clk_rate > 150000000)
280 priv->clk_csr = 0x1;
281 else
282 priv->clk_csr = 0x0;
283 }
cd7201f4
GC		 284}
285
47dd7a54
GC		 286static void print_pkt(unsigned char *buf, int len)
287{
424c4f78
AS		 288 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
289 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
47dd7a54 290}
47dd7a54 291
ce736788 292static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
47dd7a54 293{
ce736788 294 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
a6a3e026 295 u32 avail;
e3ad57c9 296
ce736788
JP		 297 if (tx_q->dirty_tx > tx_q->cur_tx)
298 avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
e3ad57c9 299 else
ce736788 300 avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1;
e3ad57c9
GC		 301
302 return avail;
303}
304
54139cf3
JP		 305/**
306 * stmmac_rx_dirty - Get RX queue dirty
307 * @priv: driver private structure
308 * @queue: RX queue index
309 */
310static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
e3ad57c9 311{
54139cf3 312 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
a6a3e026 313 u32 dirty;
e3ad57c9 314
54139cf3
JP		 315 if (rx_q->dirty_rx <= rx_q->cur_rx)
316 dirty = rx_q->cur_rx - rx_q->dirty_rx;
e3ad57c9 317 else
54139cf3 318 dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx;
e3ad57c9
GC		 319
320 return dirty;
47dd7a54
GC		 321}
322
32ceabca 323/**
732fdf0e 324 * stmmac_enable_eee_mode - check and enter in LPI mode
32ceabca 325 * @priv: driver private structure
732fdf0e
GC		 326 * Description: this function is to verify and enter in LPI mode in case of
327 * EEE.
32ceabca 328 */
d765955d
GC		 329static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
330{
ce736788
JP		 331 u32 tx_cnt = priv->plat->tx_queues_to_use;
332 u32 queue;
333
334 /* check if all TX queues have the work finished */
335 for (queue = 0; queue < tx_cnt; queue++) {
336 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
337
338 if (tx_q->dirty_tx != tx_q->cur_tx)
339 return; /* still unfinished work */
340 }
341
d765955d 342 /* Check and enter in LPI mode */
ce736788 343 if (!priv->tx_path_in_lpi_mode)
c10d4c82
JA		 344 stmmac_set_eee_mode(priv, priv->hw,
345 priv->plat->en_tx_lpi_clockgating);
d765955d
GC		 346}
347
32ceabca 348/**
732fdf0e 349 * stmmac_disable_eee_mode - disable and exit from LPI mode
32ceabca
GC		 350 * @priv: driver private structure
351 * Description: this function is to exit and disable EEE in case of
352 * LPI state is true. This is called by the xmit.
353 */
d765955d
GC		 354void stmmac_disable_eee_mode(struct stmmac_priv *priv)
355{
c10d4c82 356 stmmac_reset_eee_mode(priv, priv->hw);
d765955d
GC		 357 del_timer_sync(&priv->eee_ctrl_timer);
358 priv->tx_path_in_lpi_mode = false;
359}
360
361/**
732fdf0e 362 * stmmac_eee_ctrl_timer - EEE TX SW timer.
d765955d
GC		 363 * @arg : data hook
364 * Description:
32ceabca 365 * if there is no data transfer and if we are not in LPI state,
d765955d
GC		 366 * then MAC Transmitter can be moved to LPI state.
367 */
e99e88a9 368static void stmmac_eee_ctrl_timer(struct timer_list *t)
d765955d 369{
e99e88a9 370 struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);
d765955d
GC		 371
372 stmmac_enable_eee_mode(priv);
f5351ef7 373 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
d765955d
GC		 374}
375
376/**
732fdf0e 377 * stmmac_eee_init - init EEE
32ceabca 378 * @priv: driver private structure
d765955d 379 * Description:
732fdf0e
GC		 380 * if the GMAC supports the EEE (from the HW cap reg) and the phy device
381 * can also manage EEE, this function enable the LPI state and start related
382 * timer.
d765955d
GC		 383 */
384bool stmmac_eee_init(struct stmmac_priv *priv)
385{
74371272 386 int tx_lpi_timer = priv->tx_lpi_timer;
879626e3 387
f5351ef7
GC		 388 /* Using PCS we cannot dial with the phy registers at this stage
389 * so we do not support extra feature like EEE.
390 */
a47b9e15
DZ		 391 if (priv->hw->pcs == STMMAC_PCS_TBI ||
392 priv->hw->pcs == STMMAC_PCS_RTBI)
74371272 393 return false;
d765955d 394
74371272
JA		 395 /* Check if MAC core supports the EEE feature. */
396 if (!priv->dma_cap.eee)
397 return false;
398
399 mutex_lock(&priv->lock);
4741cf9c 400
74371272 401 /* Check if it needs to be deactivated */
177d935a
JH		 402 if (!priv->eee_active) {
403 if (priv->eee_enabled) {
404 netdev_dbg(priv->dev, "disable EEE\n");
405 del_timer_sync(&priv->eee_ctrl_timer);
406 stmmac_set_eee_timer(priv, priv->hw, 0, tx_lpi_timer);
407 }
0867bb97 408 mutex_unlock(&priv->lock);
74371272 409 return false;
d765955d 410 }
74371272
JA		 411
412 if (priv->eee_active && !priv->eee_enabled) {
413 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0);
414 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
415 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS,
416 tx_lpi_timer);
417 }
418
419 mutex_unlock(&priv->lock);
420 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
421 return true;
d765955d
GC		 422}
423
732fdf0e 424/* stmmac_get_tx_hwtstamp - get HW TX timestamps
32ceabca 425 * @priv: driver private structure
ba1ffd74 426 * @p : descriptor pointer
891434b1
RK		 427 * @skb : the socket buffer
428 * Description :
429 * This function will read timestamp from the descriptor & pass it to stack.
430 * and also perform some sanity checks.
431 */
432static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
ba1ffd74 433 struct dma_desc *p, struct sk_buff *skb)
891434b1
RK		 434{
435 struct skb_shared_hwtstamps shhwtstamp;
25e80cd0 436 bool found = false;
df103170 437 u64 ns = 0;
891434b1
RK		 438
439 if (!priv->hwts_tx_en)
440 return;
441
ceb69499 442 /* exit if skb doesn't support hw tstamp */
75e4364f 443 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
891434b1
RK		 444 return;
445
891434b1 446 /* check tx tstamp status */
42de047d 447 if (stmmac_get_tx_timestamp_status(priv, p)) {
42de047d 448 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns);
25e80cd0
JA		 449 found = true;
450 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {
451 found = true;
452 }
891434b1 453
25e80cd0 454 if (found) {
ba1ffd74
GC		 455 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
456 shhwtstamp.hwtstamp = ns_to_ktime(ns);
891434b1 457
33d4c482 458 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);
ba1ffd74
GC		 459 /* pass tstamp to stack */
460 skb_tstamp_tx(skb, &shhwtstamp);
461 }
891434b1
RK		 462}
463
732fdf0e 464/* stmmac_get_rx_hwtstamp - get HW RX timestamps
32ceabca 465 * @priv: driver private structure
ba1ffd74
GC		 466 * @p : descriptor pointer
467 * @np : next descriptor pointer
891434b1
RK		 468 * @skb : the socket buffer
469 * Description :
470 * This function will read received packet's timestamp from the descriptor
471 * and pass it to stack. It also perform some sanity checks.
472 */
ba1ffd74
GC		 473static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
474 struct dma_desc *np, struct sk_buff *skb)
891434b1
RK		 475{
476 struct skb_shared_hwtstamps *shhwtstamp = NULL;
98870943 477 struct dma_desc *desc = p;
df103170 478 u64 ns = 0;
891434b1
RK		 479
480 if (!priv->hwts_rx_en)
481 return;
98870943 482 /* For GMAC4, the valid timestamp is from CTX next desc. */
7d9e6c5a 483 if (priv->plat->has_gmac4 || priv->plat->has_xgmac)
98870943 484 desc = np;
891434b1 485
ba1ffd74 486 /* Check if timestamp is available */
42de047d
JA		 487 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) {
488 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
33d4c482 489 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
ba1ffd74
GC		 490 shhwtstamp = skb_hwtstamps(skb);
491 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
492 shhwtstamp->hwtstamp = ns_to_ktime(ns);
493 } else {
33d4c482 494 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");
ba1ffd74 495 }
891434b1
RK		 496}
497
498/**
d6228b7c 499 * stmmac_hwtstamp_set - control hardware timestamping.
891434b1 500 * @dev: device pointer.
8d45e42b 501 * @ifr: An IOCTL specific structure, that can contain a pointer to
891434b1
RK		 502 * a proprietary structure used to pass information to the driver.
503 * Description:
504 * This function configures the MAC to enable/disable both outgoing(TX)
505 * and incoming(RX) packets time stamping based on user input.
506 * Return Value:
507 * 0 on success and an appropriate -ve integer on failure.
508 */
d6228b7c 509static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
891434b1
RK		 510{
511 struct stmmac_priv *priv = netdev_priv(dev);
512 struct hwtstamp_config config;
0a624155 513 struct timespec64 now;
891434b1
RK		 514 u64 temp = 0;
515 u32 ptp_v2 = 0;
516 u32 tstamp_all = 0;
517 u32 ptp_over_ipv4_udp = 0;
518 u32 ptp_over_ipv6_udp = 0;
519 u32 ptp_over_ethernet = 0;
520 u32 snap_type_sel = 0;
521 u32 ts_master_en = 0;
522 u32 ts_event_en = 0;
df103170 523 u32 sec_inc = 0;
891434b1 524 u32 value = 0;
7d9e6c5a
JA		 525 bool xmac;
526
527 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
891434b1
RK		 528
529 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
530 netdev_alert(priv->dev, "No support for HW time stamping\n");
531 priv->hwts_tx_en = 0;
532 priv->hwts_rx_en = 0;
533
534 return -EOPNOTSUPP;
535 }
536
537 if (copy_from_user(&config, ifr->ifr_data,
d6228b7c 538 sizeof(config)))
891434b1
RK		 539 return -EFAULT;
540
38ddc59d
LC		 541 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
542 __func__, config.flags, config.tx_type, config.rx_filter);
891434b1
RK		 543
544 /* reserved for future extensions */
545 if (config.flags)
546 return -EINVAL;
547
5f3da328
BH		 548 if (config.tx_type != HWTSTAMP_TX_OFF &&
549 config.tx_type != HWTSTAMP_TX_ON)
891434b1 550 return -ERANGE;
891434b1
RK		 551
552 if (priv->adv_ts) {
553 switch (config.rx_filter) {
891434b1 554 case HWTSTAMP_FILTER_NONE:
ceb69499 555 /* time stamp no incoming packet at all */
891434b1
RK		 556 config.rx_filter = HWTSTAMP_FILTER_NONE;
557 break;
558
891434b1 559 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
ceb69499 560 /* PTP v1, UDP, any kind of event packet */
891434b1 561 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
7d8e249f
IA		 562 /* 'xmac' hardware can support Sync, Pdelay_Req and
563 * Pdelay_resp by setting bit14 and bits17/16 to 01
564 * This leaves Delay_Req timestamps out.
565 * Enable all events *and* general purpose message
566 * timestamping
567 */
568 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
891434b1
RK		 569 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
570 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
571 break;
572
891434b1 573 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
ceb69499 574 /* PTP v1, UDP, Sync packet */
891434b1
RK		 575 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
576 /* take time stamp for SYNC messages only */
577 ts_event_en = PTP_TCR_TSEVNTENA;
578
579 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
580 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
581 break;
582
891434b1 583 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
ceb69499 584 /* PTP v1, UDP, Delay_req packet */
891434b1
RK		 585 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
586 /* take time stamp for Delay_Req messages only */
587 ts_master_en = PTP_TCR_TSMSTRENA;
588 ts_event_en = PTP_TCR_TSEVNTENA;
589
590 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
591 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
592 break;
593
891434b1 594 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
ceb69499 595 /* PTP v2, UDP, any kind of event packet */
891434b1
RK		 596 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
597 ptp_v2 = PTP_TCR_TSVER2ENA;
598 /* take time stamp for all event messages */
7d8e249f 599 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
891434b1
RK		 600
601 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
602 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
603 break;
604
891434b1 605 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
ceb69499 606 /* PTP v2, UDP, Sync packet */
891434b1
RK		 607 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
608 ptp_v2 = PTP_TCR_TSVER2ENA;
609 /* take time stamp for SYNC messages only */
610 ts_event_en = PTP_TCR_TSEVNTENA;
611
612 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
613 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
614 break;
615
891434b1 616 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
ceb69499 617 /* PTP v2, UDP, Delay_req packet */
891434b1
RK		 618 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
619 ptp_v2 = PTP_TCR_TSVER2ENA;
620 /* take time stamp for Delay_Req messages only */
621 ts_master_en = PTP_TCR_TSMSTRENA;
622 ts_event_en = PTP_TCR_TSEVNTENA;
623
624 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
625 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
626 break;
627
891434b1 628 case HWTSTAMP_FILTER_PTP_V2_EVENT:
ceb69499 629 /* PTP v2/802.AS1 any layer, any kind of event packet */
891434b1
RK		 630 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
631 ptp_v2 = PTP_TCR_TSVER2ENA;
7d8e249f 632 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
14f34733 633 ts_event_en = PTP_TCR_TSEVNTENA;
891434b1
RK		 634 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
635 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
636 ptp_over_ethernet = PTP_TCR_TSIPENA;
637 break;
638
891434b1 639 case HWTSTAMP_FILTER_PTP_V2_SYNC:
ceb69499 640 /* PTP v2/802.AS1, any layer, Sync packet */
891434b1
RK		 641 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
642 ptp_v2 = PTP_TCR_TSVER2ENA;
643 /* take time stamp for SYNC messages only */
644 ts_event_en = PTP_TCR_TSEVNTENA;
645
646 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
647 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
648 ptp_over_ethernet = PTP_TCR_TSIPENA;
649 break;
650
891434b1 651 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
ceb69499 652 /* PTP v2/802.AS1, any layer, Delay_req packet */
891434b1
RK		 653 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
654 ptp_v2 = PTP_TCR_TSVER2ENA;
655 /* take time stamp for Delay_Req messages only */
656 ts_master_en = PTP_TCR_TSMSTRENA;
657 ts_event_en = PTP_TCR_TSEVNTENA;
658
659 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
660 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
661 ptp_over_ethernet = PTP_TCR_TSIPENA;
662 break;
663
e3412575 664 case HWTSTAMP_FILTER_NTP_ALL:
891434b1 665 case HWTSTAMP_FILTER_ALL:
ceb69499 666 /* time stamp any incoming packet */
891434b1
RK		 667 config.rx_filter = HWTSTAMP_FILTER_ALL;
668 tstamp_all = PTP_TCR_TSENALL;
669 break;
670
671 default:
672 return -ERANGE;
673 }
674 } else {
675 switch (config.rx_filter) {
676 case HWTSTAMP_FILTER_NONE:
677 config.rx_filter = HWTSTAMP_FILTER_NONE;
678 break;
679 default:
680 /* PTP v1, UDP, any kind of event packet */
681 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
682 break;
683 }
684 }
685 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
5f3da328 686 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
891434b1
RK		 687
688 if (!priv->hwts_tx_en && !priv->hwts_rx_en)
cc4c9001 689 stmmac_config_hw_tstamping(priv, priv->ptpaddr, 0);
891434b1
RK		 690 else {
691 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
ceb69499
GC		 692 tstamp_all | ptp_v2 | ptp_over_ethernet |
693 ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
694 ts_master_en | snap_type_sel);
cc4c9001 695 stmmac_config_hw_tstamping(priv, priv->ptpaddr, value);
891434b1
RK		 696
697 /* program Sub Second Increment reg */
cc4c9001
JA		 698 stmmac_config_sub_second_increment(priv,
699 priv->ptpaddr, priv->plat->clk_ptp_rate,
7d9e6c5a 700 xmac, &sec_inc);
19d857c9 701 temp = div_u64(1000000000ULL, sec_inc);
891434b1 702
9a8a02c9
JA		 703 /* Store sub second increment and flags for later use */
704 priv->sub_second_inc = sec_inc;
705 priv->systime_flags = value;
706
891434b1
RK		 707 /* calculate default added value:
708 * formula is :
709 * addend = (2^32)/freq_div_ratio;
19d857c9 710 * where, freq_div_ratio = 1e9ns/sec_inc
891434b1 711 */
19d857c9 712 temp = (u64)(temp << 32);
f573c0b9 713 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
cc4c9001 714 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);
891434b1
RK		 715
716 /* initialize system time */
0a624155
AB		 717 ktime_get_real_ts64(&now);
718
719 /* lower 32 bits of tv_sec are safe until y2106 */
cc4c9001
JA		 720 stmmac_init_systime(priv, priv->ptpaddr,
721 (u32)now.tv_sec, now.tv_nsec);
891434b1
RK		 722 }
723
d6228b7c
AP		 724 memcpy(&priv->tstamp_config, &config, sizeof(config));
725
891434b1 726 return copy_to_user(ifr->ifr_data, &config,
d6228b7c
AP		 727 sizeof(config)) ? -EFAULT : 0;
728}
729
730/**
731 * stmmac_hwtstamp_get - read hardware timestamping.
732 * @dev: device pointer.
733 * @ifr: An IOCTL specific structure, that can contain a pointer to
734 * a proprietary structure used to pass information to the driver.
735 * Description:
736 * This function obtain the current hardware timestamping settings
737 as requested.
738 */
739static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
740{
741 struct stmmac_priv *priv = netdev_priv(dev);
742 struct hwtstamp_config *config = &priv->tstamp_config;
743
744 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
745 return -EOPNOTSUPP;
746
747 return copy_to_user(ifr->ifr_data, config,
748 sizeof(*config)) ? -EFAULT : 0;
891434b1
RK		 749}
750
32ceabca 751/**
732fdf0e 752 * stmmac_init_ptp - init PTP
32ceabca 753 * @priv: driver private structure
732fdf0e 754 * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
32ceabca 755 * This is done by looking at the HW cap. register.
732fdf0e 756 * This function also registers the ptp driver.
32ceabca 757 */
92ba6888 758static int stmmac_init_ptp(struct stmmac_priv *priv)
891434b1 759{
7d9e6c5a
JA		 760 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
761
92ba6888
RK		 762 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
763 return -EOPNOTSUPP;
764
7cd01399 765 priv->adv_ts = 0;
7d9e6c5a
JA		 766 /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */
767 if (xmac && priv->dma_cap.atime_stamp)
be9b3174
GC		 768 priv->adv_ts = 1;
769 /* Dwmac 3.x core with extend_desc can support adv_ts */
770 else if (priv->extend_desc && priv->dma_cap.atime_stamp)
7cd01399
VB		 771 priv->adv_ts = 1;
772
be9b3174
GC		 773 if (priv->dma_cap.time_stamp)
774 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
7cd01399 775
be9b3174
GC		 776 if (priv->adv_ts)
777 netdev_info(priv->dev,
778 "IEEE 1588-2008 Advanced Timestamp supported\n");
891434b1 779
891434b1
RK		 780 priv->hwts_tx_en = 0;
781 priv->hwts_rx_en = 0;
92ba6888 782
c30a70d3
GC		 783 stmmac_ptp_register(priv);
784
785 return 0;
92ba6888
RK		 786}
787
788static void stmmac_release_ptp(struct stmmac_priv *priv)
789{
f573c0b9 790 if (priv->plat->clk_ptp_ref)
791 clk_disable_unprepare(priv->plat->clk_ptp_ref);
92ba6888 792 stmmac_ptp_unregister(priv);
891434b1
RK		 793}
794
29feff39
JP		 795/**
796 * stmmac_mac_flow_ctrl - Configure flow control in all queues
797 * @priv: driver private structure
798 * Description: It is used for configuring the flow control in all queues
799 */
800static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
801{
802 u32 tx_cnt = priv->plat->tx_queues_to_use;
803
c10d4c82
JA		 804 stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl,
805 priv->pause, tx_cnt);
29feff39
JP		 806}
807
eeef2f6b
JA		 808static void stmmac_validate(struct phylink_config *config,
809 unsigned long *supported,
810 struct phylink_link_state *state)
811{
812 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
5b0d7d7d 813 __ETHTOOL_DECLARE_LINK_MODE_MASK(mac_supported) = { 0, };
eeef2f6b
JA		 814 __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
815 int tx_cnt = priv->plat->tx_queues_to_use;
816 int max_speed = priv->plat->max_speed;
817
5b0d7d7d
JA		 818 phylink_set(mac_supported, 10baseT_Half);
819 phylink_set(mac_supported, 10baseT_Full);
820 phylink_set(mac_supported, 100baseT_Half);
821 phylink_set(mac_supported, 100baseT_Full);
df7699c7
JA		 822 phylink_set(mac_supported, 1000baseT_Half);
823 phylink_set(mac_supported, 1000baseT_Full);
824 phylink_set(mac_supported, 1000baseKX_Full);
5b0d7d7d
JA		 825
826 phylink_set(mac_supported, Autoneg);
827 phylink_set(mac_supported, Pause);
828 phylink_set(mac_supported, Asym_Pause);
829 phylink_set_port_modes(mac_supported);
830
eeef2f6b
JA		 831 /* Cut down 1G if asked to */
832 if ((max_speed > 0) && (max_speed < 1000)) {
833 phylink_set(mask, 1000baseT_Full);
834 phylink_set(mask, 1000baseX_Full);
5b0d7d7d 835 } else if (priv->plat->has_xgmac) {
d9da2c87
JA		 836 if (!max_speed || (max_speed >= 2500)) {
837 phylink_set(mac_supported, 2500baseT_Full);
838 phylink_set(mac_supported, 2500baseX_Full);
839 }
840 if (!max_speed || (max_speed >= 5000)) {
841 phylink_set(mac_supported, 5000baseT_Full);
842 }
843 if (!max_speed || (max_speed >= 10000)) {
844 phylink_set(mac_supported, 10000baseSR_Full);
845 phylink_set(mac_supported, 10000baseLR_Full);
846 phylink_set(mac_supported, 10000baseER_Full);
847 phylink_set(mac_supported, 10000baseLRM_Full);
848 phylink_set(mac_supported, 10000baseT_Full);
849 phylink_set(mac_supported, 10000baseKX4_Full);
850 phylink_set(mac_supported, 10000baseKR_Full);
851 }
8a880936
JA		 852 if (!max_speed || (max_speed >= 25000)) {
853 phylink_set(mac_supported, 25000baseCR_Full);
854 phylink_set(mac_supported, 25000baseKR_Full);
855 phylink_set(mac_supported, 25000baseSR_Full);
856 }
857 if (!max_speed || (max_speed >= 40000)) {
858 phylink_set(mac_supported, 40000baseKR4_Full);
859 phylink_set(mac_supported, 40000baseCR4_Full);
860 phylink_set(mac_supported, 40000baseSR4_Full);
861 phylink_set(mac_supported, 40000baseLR4_Full);
862 }
863 if (!max_speed || (max_speed >= 50000)) {
864 phylink_set(mac_supported, 50000baseCR2_Full);
865 phylink_set(mac_supported, 50000baseKR2_Full);
866 phylink_set(mac_supported, 50000baseSR2_Full);
867 phylink_set(mac_supported, 50000baseKR_Full);
868 phylink_set(mac_supported, 50000baseSR_Full);
869 phylink_set(mac_supported, 50000baseCR_Full);
870 phylink_set(mac_supported, 50000baseLR_ER_FR_Full);
871 phylink_set(mac_supported, 50000baseDR_Full);
872 }
873 if (!max_speed || (max_speed >= 100000)) {
874 phylink_set(mac_supported, 100000baseKR4_Full);
875 phylink_set(mac_supported, 100000baseSR4_Full);
876 phylink_set(mac_supported, 100000baseCR4_Full);
877 phylink_set(mac_supported, 100000baseLR4_ER4_Full);
878 phylink_set(mac_supported, 100000baseKR2_Full);
879 phylink_set(mac_supported, 100000baseSR2_Full);
880 phylink_set(mac_supported, 100000baseCR2_Full);
881 phylink_set(mac_supported, 100000baseLR2_ER2_FR2_Full);
882 phylink_set(mac_supported, 100000baseDR2_Full);
883 }
eeef2f6b
JA		 884 }
885
886 /* Half-Duplex can only work with single queue */
887 if (tx_cnt > 1) {
888 phylink_set(mask, 10baseT_Half);
889 phylink_set(mask, 100baseT_Half);
890 phylink_set(mask, 1000baseT_Half);
891 }
892
422829f9
JA		 893 linkmode_and(supported, supported, mac_supported);
894 linkmode_andnot(supported, supported, mask);
895
896 linkmode_and(state->advertising, state->advertising, mac_supported);
897 linkmode_andnot(state->advertising, state->advertising, mask);
f213bbe8
JA		 898
899 /* If PCS is supported, check which modes it supports. */
900 stmmac_xpcs_validate(priv, &priv->hw->xpcs_args, supported, state);
eeef2f6b
JA		 901}
902
d46b7e4f
RK		 903static void stmmac_mac_pcs_get_state(struct phylink_config *config,
904 struct phylink_link_state *state)
eeef2f6b 905{
f213bbe8
JA		 906 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
907
d46b7e4f 908 state->link = 0;
f213bbe8 909 stmmac_xpcs_get_state(priv, &priv->hw->xpcs_args, state);
eeef2f6b
JA		 910}
911
74371272
JA		 912static void stmmac_mac_config(struct phylink_config *config, unsigned int mode,
913 const struct phylink_link_state *state)
46f69ded 914{
f213bbe8
JA		 915 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
916
917 stmmac_xpcs_config(priv, &priv->hw->xpcs_args, state);
46f69ded
JA		 918}
919
920static void stmmac_mac_an_restart(struct phylink_config *config)
921{
922 /* Not Supported */
923}
924
925static void stmmac_mac_link_down(struct phylink_config *config,
926 unsigned int mode, phy_interface_t interface)
927{
928 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
929
930 stmmac_mac_set(priv, priv->ioaddr, false);
931 priv->eee_active = false;
932 stmmac_eee_init(priv);
933 stmmac_set_eee_pls(priv, priv->hw, false);
934}
935
936static void stmmac_mac_link_up(struct phylink_config *config,
937 struct phy_device *phy,
938 unsigned int mode, phy_interface_t interface,
939 int speed, int duplex,
940 bool tx_pause, bool rx_pause)
9ad372fc 941{
74371272 942 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
9ad372fc
JA		 943 u32 ctrl;
944
f213bbe8
JA		 945 stmmac_xpcs_link_up(priv, &priv->hw->xpcs_args, speed, interface);
946
9ad372fc 947 ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
74371272 948 ctrl &= ~priv->hw->link.speed_mask;
9ad372fc 949
46f69ded
JA		 950 if (interface == PHY_INTERFACE_MODE_USXGMII) {
951 switch (speed) {
5b0d7d7d
JA		 952 case SPEED_10000:
953 ctrl |= priv->hw->link.xgmii.speed10000;
954 break;
955 case SPEED_5000:
956 ctrl |= priv->hw->link.xgmii.speed5000;
957 break;
958 case SPEED_2500:
959 ctrl |= priv->hw->link.xgmii.speed2500;
960 break;
961 default:
962 return;
963 }
8a880936
JA		 964 } else if (interface == PHY_INTERFACE_MODE_XLGMII) {
965 switch (speed) {
966 case SPEED_100000:
967 ctrl |= priv->hw->link.xlgmii.speed100000;
968 break;
969 case SPEED_50000:
970 ctrl |= priv->hw->link.xlgmii.speed50000;
971 break;
972 case SPEED_40000:
973 ctrl |= priv->hw->link.xlgmii.speed40000;
974 break;
975 case SPEED_25000:
976 ctrl |= priv->hw->link.xlgmii.speed25000;
977 break;
978 case SPEED_10000:
979 ctrl |= priv->hw->link.xgmii.speed10000;
980 break;
981 case SPEED_2500:
982 ctrl |= priv->hw->link.speed2500;
983 break;
984 case SPEED_1000:
985 ctrl |= priv->hw->link.speed1000;
986 break;
987 default:
988 return;
989 }
5b0d7d7d 990 } else {
46f69ded 991 switch (speed) {
5b0d7d7d
JA		 992 case SPEED_2500:
993 ctrl |= priv->hw->link.speed2500;
994 break;
995 case SPEED_1000:
996 ctrl |= priv->hw->link.speed1000;
997 break;
998 case SPEED_100:
999 ctrl |= priv->hw->link.speed100;
1000 break;
1001 case SPEED_10:
1002 ctrl |= priv->hw->link.speed10;
1003 break;
1004 default:
1005 return;
1006 }
9ad372fc
JA		 1007 }
1008
46f69ded 1009 priv->speed = speed;
9ad372fc 1010
74371272 1011 if (priv->plat->fix_mac_speed)
46f69ded 1012 priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed);
74371272 1013
46f69ded 1014 if (!duplex)
74371272
JA		 1015 ctrl &= ~priv->hw->link.duplex;
1016 else
1017 ctrl |= priv->hw->link.duplex;
9ad372fc
JA		 1018
1019 /* Flow Control operation */
46f69ded
JA		 1020 if (tx_pause && rx_pause)
1021 stmmac_mac_flow_ctrl(priv, duplex);
9ad372fc
JA		 1022
1023 writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
9ad372fc
JA		 1024
1025 stmmac_mac_set(priv, priv->ioaddr, true);
5b111770 1026 if (phy && priv->dma_cap.eee) {
74371272
JA		 1027 priv->eee_active = phy_init_eee(phy, 1) >= 0;
1028 priv->eee_enabled = stmmac_eee_init(priv);
1029 stmmac_set_eee_pls(priv, priv->hw, true);
1030 }
9ad372fc
JA		 1031}
1032
74371272 1033static const struct phylink_mac_ops stmmac_phylink_mac_ops = {
eeef2f6b 1034 .validate = stmmac_validate,
d46b7e4f 1035 .mac_pcs_get_state = stmmac_mac_pcs_get_state,
74371272 1036 .mac_config = stmmac_mac_config,
eeef2f6b 1037 .mac_an_restart = stmmac_mac_an_restart,
74371272
JA		 1038 .mac_link_down = stmmac_mac_link_down,
1039 .mac_link_up = stmmac_mac_link_up,
eeef2f6b
JA		 1040};
1041
32ceabca 1042/**
732fdf0e 1043 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
32ceabca
GC		 1044 * @priv: driver private structure
1045 * Description: this is to verify if the HW supports the PCS.
1046 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
1047 * configured for the TBI, RTBI, or SGMII PHY interface.
1048 */
e58bb43f
GC		 1049static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
1050{
1051 int interface = priv->plat->interface;
1052
1053 if (priv->dma_cap.pcs) {
0d909dcd
BA		 1054 if ((interface == PHY_INTERFACE_MODE_RGMII) ||
1055 (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
1056 (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
1057 (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
38ddc59d 1058 netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
3fe5cadb 1059 priv->hw->pcs = STMMAC_PCS_RGMII;
0d909dcd 1060 } else if (interface == PHY_INTERFACE_MODE_SGMII) {
38ddc59d 1061 netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
3fe5cadb 1062 priv->hw->pcs = STMMAC_PCS_SGMII;
e58bb43f
GC		 1063 }
1064 }
1065}
1066
47dd7a54
GC		 1067/**
1068 * stmmac_init_phy - PHY initialization
1069 * @dev: net device structure
1070 * Description: it initializes the driver's PHY state, and attaches the PHY
1071 * to the mac driver.
1072 * Return value:
1073 * 0 on success
1074 */
1075static int stmmac_init_phy(struct net_device *dev)
1076{
1077 struct stmmac_priv *priv = netdev_priv(dev);
74371272
JA		 1078 struct device_node *node;
1079 int ret;
5790cf3c 1080
4838a540 1081 node = priv->plat->phylink_node;
5790cf3c 1082
42e87024 1083 if (node)
74371272 1084 ret = phylink_of_phy_connect(priv->phylink, node, 0);
42e87024
JA		 1085
1086 /* Some DT bindings do not set-up the PHY handle. Let's try to
1087 * manually parse it
1088 */
1089 if (!node || ret) {
74371272
JA		 1090 int addr = priv->plat->phy_addr;
1091 struct phy_device *phydev;
47dd7a54 1092
74371272
JA		 1093 phydev = mdiobus_get_phy(priv->mii, addr);
1094 if (!phydev) {
1095 netdev_err(priv->dev, "no phy at addr %d\n", addr);
dfc50fca 1096 return -ENODEV;
74371272 1097 }
dfc50fca 1098
74371272 1099 ret = phylink_connect_phy(priv->phylink, phydev);
47dd7a54
GC		 1100 }
1101
74371272
JA		 1102 return ret;
1103}
79ee1dc3 1104
74371272
JA		 1105static int stmmac_phy_setup(struct stmmac_priv *priv)
1106{
c63d1e5c 1107 struct fwnode_handle *fwnode = of_fwnode_handle(priv->plat->phylink_node);
0060c878 1108 int mode = priv->plat->phy_interface;
74371272 1109 struct phylink *phylink;
b6cfffa7 1110
74371272
JA		 1111 priv->phylink_config.dev = &priv->dev->dev;
1112 priv->phylink_config.type = PHYLINK_NETDEV;
f213bbe8 1113 priv->phylink_config.pcs_poll = true;
8e99fc5f 1114
8dc6051c
JA		 1115 if (!fwnode)
1116 fwnode = dev_fwnode(priv->device);
1117
c63d1e5c 1118 phylink = phylink_create(&priv->phylink_config, fwnode,
74371272
JA		 1119 mode, &stmmac_phylink_mac_ops);
1120 if (IS_ERR(phylink))
1121 return PTR_ERR(phylink);
c51e424d 1122
74371272 1123 priv->phylink = phylink;
47dd7a54
GC		 1124 return 0;
1125}
1126
71fedb01 1127static void stmmac_display_rx_rings(struct stmmac_priv *priv)
c24602ef 1128{
54139cf3 1129 u32 rx_cnt = priv->plat->rx_queues_to_use;
71fedb01 1130 void *head_rx;
54139cf3 1131 u32 queue;
aff3d9ef 1132
54139cf3
JP		 1133 /* Display RX rings */
1134 for (queue = 0; queue < rx_cnt; queue++) {
1135 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
d0225e7d 1136
54139cf3
JP		 1137 pr_info("\tRX Queue %u rings\n", queue);
1138
1139 if (priv->extend_desc)
1140 head_rx = (void *)rx_q->dma_erx;
1141 else
1142 head_rx = (void *)rx_q->dma_rx;
1143
1144 /* Display RX ring */
42de047d 1145 stmmac_display_ring(priv, head_rx, DMA_RX_SIZE, true);
54139cf3 1146 }
71fedb01
JP		 1147}
1148
1149static void stmmac_display_tx_rings(struct stmmac_priv *priv)
1150{
ce736788 1151 u32 tx_cnt = priv->plat->tx_queues_to_use;
71fedb01 1152 void *head_tx;
ce736788 1153 u32 queue;
71fedb01 1154
ce736788
JP		 1155 /* Display TX rings */
1156 for (queue = 0; queue < tx_cnt; queue++) {
1157 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
71fedb01 1158
ce736788
JP		 1159 pr_info("\tTX Queue %d rings\n", queue);
1160
1161 if (priv->extend_desc)
1162 head_tx = (void *)tx_q->dma_etx;
579a25a8
JA		 1163 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1164 head_tx = (void *)tx_q->dma_entx;
ce736788
JP		 1165 else
1166 head_tx = (void *)tx_q->dma_tx;
1167
42de047d 1168 stmmac_display_ring(priv, head_tx, DMA_TX_SIZE, false);
ce736788 1169 }
c24602ef
GC		 1170}
1171
71fedb01
JP		 1172static void stmmac_display_rings(struct stmmac_priv *priv)
1173{
1174 /* Display RX ring */
1175 stmmac_display_rx_rings(priv);
1176
1177 /* Display TX ring */
1178 stmmac_display_tx_rings(priv);
1179}
1180
286a8372
GC		 1181static int stmmac_set_bfsize(int mtu, int bufsize)
1182{
1183 int ret = bufsize;
1184
b2f3a481
JA		 1185 if (mtu >= BUF_SIZE_8KiB)
1186 ret = BUF_SIZE_16KiB;
1187 else if (mtu >= BUF_SIZE_4KiB)
286a8372
GC		 1188 ret = BUF_SIZE_8KiB;
1189 else if (mtu >= BUF_SIZE_2KiB)
1190 ret = BUF_SIZE_4KiB;
d916701c 1191 else if (mtu > DEFAULT_BUFSIZE)
286a8372
GC		 1192 ret = BUF_SIZE_2KiB;
1193 else
d916701c 1194 ret = DEFAULT_BUFSIZE;
286a8372
GC		 1195
1196 return ret;
1197}
1198
32ceabca 1199/**
71fedb01 1200 * stmmac_clear_rx_descriptors - clear RX descriptors
32ceabca 1201 * @priv: driver private structure
54139cf3 1202 * @queue: RX queue index
71fedb01 1203 * Description: this function is called to clear the RX descriptors
32ceabca
GC		 1204 * in case of both basic and extended descriptors are used.
1205 */
54139cf3 1206static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
c24602ef 1207{
54139cf3 1208 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
5bacd778 1209 int i;
c24602ef 1210
71fedb01 1211 /* Clear the RX descriptors */
e3ad57c9 1212 for (i = 0; i < DMA_RX_SIZE; i++)
c24602ef 1213 if (priv->extend_desc)
42de047d
JA		 1214 stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic,
1215 priv->use_riwt, priv->mode,
583e6361
AK		 1216 (i == DMA_RX_SIZE - 1),
1217 priv->dma_buf_sz);
c24602ef 1218 else
42de047d
JA		 1219 stmmac_init_rx_desc(priv, &rx_q->dma_rx[i],
1220 priv->use_riwt, priv->mode,
583e6361
AK		 1221 (i == DMA_RX_SIZE - 1),
1222 priv->dma_buf_sz);
71fedb01
JP		 1223}
1224
1225/**
1226 * stmmac_clear_tx_descriptors - clear tx descriptors
1227 * @priv: driver private structure
ce736788 1228 * @queue: TX queue index.
71fedb01
JP		 1229 * Description: this function is called to clear the TX descriptors
1230 * in case of both basic and extended descriptors are used.
1231 */
ce736788 1232static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
71fedb01 1233{
ce736788 1234 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
71fedb01
JP		 1235 int i;
1236
1237 /* Clear the TX descriptors */
579a25a8
JA		 1238 for (i = 0; i < DMA_TX_SIZE; i++) {
1239 int last = (i == (DMA_TX_SIZE - 1));
1240 struct dma_desc *p;
1241
c24602ef 1242 if (priv->extend_desc)
579a25a8
JA		 1243 p = &tx_q->dma_etx[i].basic;
1244 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1245 p = &tx_q->dma_entx[i].basic;
c24602ef 1246 else
579a25a8
JA		 1247 p = &tx_q->dma_tx[i];
1248
1249 stmmac_init_tx_desc(priv, p, priv->mode, last);
1250 }
c24602ef
GC		 1251}
1252
71fedb01
JP		 1253/**
1254 * stmmac_clear_descriptors - clear descriptors
1255 * @priv: driver private structure
1256 * Description: this function is called to clear the TX and RX descriptors
1257 * in case of both basic and extended descriptors are used.
1258 */
1259static void stmmac_clear_descriptors(struct stmmac_priv *priv)
1260{
54139cf3 1261 u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
ce736788 1262 u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
54139cf3
JP		 1263 u32 queue;
1264
71fedb01 1265 /* Clear the RX descriptors */
54139cf3
JP		 1266 for (queue = 0; queue < rx_queue_cnt; queue++)
1267 stmmac_clear_rx_descriptors(priv, queue);
71fedb01
JP		 1268
1269 /* Clear the TX descriptors */
ce736788
JP		 1270 for (queue = 0; queue < tx_queue_cnt; queue++)
1271 stmmac_clear_tx_descriptors(priv, queue);
71fedb01
JP		 1272}
1273
732fdf0e
GC		 1274/**
1275 * stmmac_init_rx_buffers - init the RX descriptor buffer.
1276 * @priv: driver private structure
1277 * @p: descriptor pointer
1278 * @i: descriptor index
54139cf3
JP		 1279 * @flags: gfp flag
1280 * @queue: RX queue index
732fdf0e
GC		 1281 * Description: this function is called to allocate a receive buffer, perform
1282 * the DMA mapping and init the descriptor.
1283 */
c24602ef 1284static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
54139cf3 1285 int i, gfp_t flags, u32 queue)
c24602ef 1286{
54139cf3 1287 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
2af6106a 1288 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
c24602ef 1289
2af6106a
JA		 1290 buf->page = page_pool_dev_alloc_pages(rx_q->page_pool);
1291 if (!buf->page)
56329137 1292 return -ENOMEM;
c24602ef 1293
67afd6d1
JA		 1294 if (priv->sph) {
1295 buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool);
1296 if (!buf->sec_page)
1297 return -ENOMEM;
1298
1299 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
1300 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr);
1301 } else {
1302 buf->sec_page = NULL;
1303 }
1304
2af6106a
JA		 1305 buf->addr = page_pool_get_dma_addr(buf->page);
1306 stmmac_set_desc_addr(priv, p, buf->addr);
2c520b1c
JA		 1307 if (priv->dma_buf_sz == BUF_SIZE_16KiB)
1308 stmmac_init_desc3(priv, p);
c24602ef
GC		 1309
1310 return 0;
1311}
1312
71fedb01
JP		 1313/**
1314 * stmmac_free_rx_buffer - free RX dma buffers
1315 * @priv: private structure
54139cf3 1316 * @queue: RX queue index
71fedb01
JP		 1317 * @i: buffer index.
1318 */
54139cf3 1319static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i)
56329137 1320{
54139cf3 1321 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
2af6106a 1322 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
54139cf3 1323
2af6106a 1324 if (buf->page)
458de8a9 1325 page_pool_put_full_page(rx_q->page_pool, buf->page, false);
2af6106a 1326 buf->page = NULL;
67afd6d1
JA		 1327
1328 if (buf->sec_page)
458de8a9 1329 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false);
67afd6d1 1330 buf->sec_page = NULL;
aff3d9ef
JP		 1331}
1332
1333/**
71fedb01
JP		 1334 * stmmac_free_tx_buffer - free RX dma buffers
1335 * @priv: private structure
ce736788 1336 * @queue: RX queue index
71fedb01
JP		 1337 * @i: buffer index.
1338 */
ce736788 1339static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i)
71fedb01 1340{
ce736788
JP		 1341 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1342
1343 if (tx_q->tx_skbuff_dma[i].buf) {
1344 if (tx_q->tx_skbuff_dma[i].map_as_page)
71fedb01 1345 dma_unmap_page(priv->device,
ce736788
JP		 1346 tx_q->tx_skbuff_dma[i].buf,
1347 tx_q->tx_skbuff_dma[i].len,
71fedb01
JP		 1348 DMA_TO_DEVICE);
1349 else
1350 dma_unmap_single(priv->device,
ce736788
JP		 1351 tx_q->tx_skbuff_dma[i].buf,
1352 tx_q->tx_skbuff_dma[i].len,
71fedb01
JP		 1353 DMA_TO_DEVICE);
1354 }
1355
ce736788
JP		 1356 if (tx_q->tx_skbuff[i]) {
1357 dev_kfree_skb_any(tx_q->tx_skbuff[i]);
1358 tx_q->tx_skbuff[i] = NULL;
1359 tx_q->tx_skbuff_dma[i].buf = 0;
1360 tx_q->tx_skbuff_dma[i].map_as_page = false;
71fedb01
JP		 1361 }
1362}
1363
1364/**
1365 * init_dma_rx_desc_rings - init the RX descriptor rings
47dd7a54 1366 * @dev: net device structure
732fdf0e 1367 * @flags: gfp flag.
71fedb01 1368 * Description: this function initializes the DMA RX descriptors
5bacd778 1369 * and allocates the socket buffers. It supports the chained and ring
286a8372 1370 * modes.
47dd7a54 1371 */
71fedb01 1372static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
47dd7a54 1373{
47dd7a54 1374 struct stmmac_priv *priv = netdev_priv(dev);
54139cf3 1375 u32 rx_count = priv->plat->rx_queues_to_use;
56329137 1376 int ret = -ENOMEM;
1d3028f4 1377 int queue;
54139cf3 1378 int i;
47dd7a54 1379
54139cf3 1380 /* RX INITIALIZATION */
b3e51069
LC		 1381 netif_dbg(priv, probe, priv->dev,
1382 "SKB addresses:\nskb\t\tskb data\tdma data\n");
47dd7a54 1383
54139cf3
JP		 1384 for (queue = 0; queue < rx_count; queue++) {
1385 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
c24602ef 1386
54139cf3
JP		 1387 netif_dbg(priv, probe, priv->dev,
1388 "(%s) dma_rx_phy=0x%08x\n", __func__,
1389 (u32)rx_q->dma_rx_phy);
f748be53 1390
cbcf0999
JA		 1391 stmmac_clear_rx_descriptors(priv, queue);
1392
54139cf3
JP		 1393 for (i = 0; i < DMA_RX_SIZE; i++) {
1394 struct dma_desc *p;
aff3d9ef 1395
54139cf3
JP		 1396 if (priv->extend_desc)
1397 p = &((rx_q->dma_erx + i)->basic);
1398 else
1399 p = rx_q->dma_rx + i;
1400
1401 ret = stmmac_init_rx_buffers(priv, p, i, flags,
1402 queue);
1403 if (ret)
1404 goto err_init_rx_buffers;
54139cf3
JP		 1405 }
1406
1407 rx_q->cur_rx = 0;
1408 rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE);
1409
54139cf3
JP		 1410 /* Setup the chained descriptor addresses */
1411 if (priv->mode == STMMAC_CHAIN_MODE) {
1412 if (priv->extend_desc)
2c520b1c
JA		 1413 stmmac_mode_init(priv, rx_q->dma_erx,
1414 rx_q->dma_rx_phy, DMA_RX_SIZE, 1);
54139cf3 1415 else
2c520b1c
JA		 1416 stmmac_mode_init(priv, rx_q->dma_rx,
1417 rx_q->dma_rx_phy, DMA_RX_SIZE, 0);
54139cf3 1418 }
71fedb01
JP		 1419 }
1420
1421 return 0;
54139cf3 1422
71fedb01 1423err_init_rx_buffers:
54139cf3
JP		 1424 while (queue >= 0) {
1425 while (--i >= 0)
1426 stmmac_free_rx_buffer(priv, queue, i);
1427
1428 if (queue == 0)
1429 break;
1430
1431 i = DMA_RX_SIZE;
1432 queue--;
1433 }
1434
71fedb01
JP		 1435 return ret;
1436}
1437
1438/**
1439 * init_dma_tx_desc_rings - init the TX descriptor rings
1440 * @dev: net device structure.
1441 * Description: this function initializes the DMA TX descriptors
1442 * and allocates the socket buffers. It supports the chained and ring
1443 * modes.
1444 */
1445static int init_dma_tx_desc_rings(struct net_device *dev)
1446{
1447 struct stmmac_priv *priv = netdev_priv(dev);
ce736788
JP		 1448 u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1449 u32 queue;
71fedb01
JP		 1450 int i;
1451
ce736788
JP		 1452 for (queue = 0; queue < tx_queue_cnt; queue++) {
1453 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
71fedb01 1454
ce736788
JP		 1455 netif_dbg(priv, probe, priv->dev,
1456 "(%s) dma_tx_phy=0x%08x\n", __func__,
1457 (u32)tx_q->dma_tx_phy);
f748be53 1458
ce736788
JP		 1459 /* Setup the chained descriptor addresses */
1460 if (priv->mode == STMMAC_CHAIN_MODE) {
1461 if (priv->extend_desc)
2c520b1c
JA		 1462 stmmac_mode_init(priv, tx_q->dma_etx,
1463 tx_q->dma_tx_phy, DMA_TX_SIZE, 1);
579a25a8 1464 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL))
2c520b1c
JA		 1465 stmmac_mode_init(priv, tx_q->dma_tx,
1466 tx_q->dma_tx_phy, DMA_TX_SIZE, 0);
ce736788 1467 }
aff3d9ef 1468
ce736788
JP		 1469 for (i = 0; i < DMA_TX_SIZE; i++) {
1470 struct dma_desc *p;
ce736788
JP		 1471 if (priv->extend_desc)
1472 p = &((tx_q->dma_etx + i)->basic);
579a25a8
JA		 1473 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1474 p = &((tx_q->dma_entx + i)->basic);
ce736788
JP		 1475 else
1476 p = tx_q->dma_tx + i;
1477
44c67f85 1478 stmmac_clear_desc(priv, p);
ce736788
JP		 1479
1480 tx_q->tx_skbuff_dma[i].buf = 0;
1481 tx_q->tx_skbuff_dma[i].map_as_page = false;
1482 tx_q->tx_skbuff_dma[i].len = 0;
1483 tx_q->tx_skbuff_dma[i].last_segment = false;
1484 tx_q->tx_skbuff[i] = NULL;
5bacd778 1485 }
aff3d9ef 1486
ce736788
JP		 1487 tx_q->dirty_tx = 0;
1488 tx_q->cur_tx = 0;
8d212a9e 1489 tx_q->mss = 0;
286a8372 1490
c22a3f48
JP		 1491 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
1492 }
aff3d9ef 1493
71fedb01
JP		 1494 return 0;
1495}
1496
1497/**
1498 * init_dma_desc_rings - init the RX/TX descriptor rings
1499 * @dev: net device structure
1500 * @flags: gfp flag.
1501 * Description: this function initializes the DMA RX/TX descriptors
1502 * and allocates the socket buffers. It supports the chained and ring
1503 * modes.
1504 */
1505static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
1506{
1507 struct stmmac_priv *priv = netdev_priv(dev);
1508 int ret;
1509
1510 ret = init_dma_rx_desc_rings(dev, flags);
1511 if (ret)
1512 return ret;
1513
1514 ret = init_dma_tx_desc_rings(dev);
1515
5bacd778 1516 stmmac_clear_descriptors(priv);
47dd7a54 1517
c24602ef
GC		 1518 if (netif_msg_hw(priv))
1519 stmmac_display_rings(priv);
56329137 1520
56329137 1521 return ret;
47dd7a54
GC		 1522}
1523
71fedb01
JP		 1524/**
1525 * dma_free_rx_skbufs - free RX dma buffers
1526 * @priv: private structure
54139cf3 1527 * @queue: RX queue index
71fedb01 1528 */
54139cf3 1529static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue)
47dd7a54
GC		 1530{
1531 int i;
1532
e3ad57c9 1533 for (i = 0; i < DMA_RX_SIZE; i++)
54139cf3 1534 stmmac_free_rx_buffer(priv, queue, i);
47dd7a54
GC		 1535}
1536
71fedb01
JP		 1537/**
1538 * dma_free_tx_skbufs - free TX dma buffers
1539 * @priv: private structure
ce736788 1540 * @queue: TX queue index
71fedb01 1541 */
ce736788 1542static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue)
47dd7a54
GC		 1543{
1544 int i;
1545
71fedb01 1546 for (i = 0; i < DMA_TX_SIZE; i++)
ce736788 1547 stmmac_free_tx_buffer(priv, queue, i);
47dd7a54
GC		 1548}
1549
54139cf3
JP		 1550/**
1551 * free_dma_rx_desc_resources - free RX dma desc resources
1552 * @priv: private structure
1553 */
1554static void free_dma_rx_desc_resources(struct stmmac_priv *priv)
1555{
1556 u32 rx_count = priv->plat->rx_queues_to_use;
1557 u32 queue;
1558
1559 /* Free RX queue resources */
1560 for (queue = 0; queue < rx_count; queue++) {
1561 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1562
1563 /* Release the DMA RX socket buffers */
1564 dma_free_rx_skbufs(priv, queue);
1565
1566 /* Free DMA regions of consistent memory previously allocated */
1567 if (!priv->extend_desc)
1568 dma_free_coherent(priv->device,
1569 DMA_RX_SIZE * sizeof(struct dma_desc),
1570 rx_q->dma_rx, rx_q->dma_rx_phy);
1571 else
1572 dma_free_coherent(priv->device, DMA_RX_SIZE *
1573 sizeof(struct dma_extended_desc),
1574 rx_q->dma_erx, rx_q->dma_rx_phy);
1575
2af6106a 1576 kfree(rx_q->buf_pool);
c3f812ce 1577 if (rx_q->page_pool)
2af6106a 1578 page_pool_destroy(rx_q->page_pool);
54139cf3
JP		 1579 }
1580}
1581
ce736788
JP		 1582/**
1583 * free_dma_tx_desc_resources - free TX dma desc resources
1584 * @priv: private structure
1585 */
1586static void free_dma_tx_desc_resources(struct stmmac_priv *priv)
1587{
1588 u32 tx_count = priv->plat->tx_queues_to_use;
62242260 1589 u32 queue;
ce736788
JP		 1590
1591 /* Free TX queue resources */
1592 for (queue = 0; queue < tx_count; queue++) {
1593 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
579a25a8
JA		 1594 size_t size;
1595 void *addr;
ce736788
JP		 1596
1597 /* Release the DMA TX socket buffers */
1598 dma_free_tx_skbufs(priv, queue);
1599
579a25a8
JA		 1600 if (priv->extend_desc) {
1601 size = sizeof(struct dma_extended_desc);
1602 addr = tx_q->dma_etx;
1603 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
1604 size = sizeof(struct dma_edesc);
1605 addr = tx_q->dma_entx;
1606 } else {
1607 size = sizeof(struct dma_desc);
1608 addr = tx_q->dma_tx;
1609 }
1610
1611 size *= DMA_TX_SIZE;
1612
1613 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy);
ce736788
JP		 1614
1615 kfree(tx_q->tx_skbuff_dma);
1616 kfree(tx_q->tx_skbuff);
1617 }
1618}
1619
732fdf0e 1620/**
71fedb01 1621 * alloc_dma_rx_desc_resources - alloc RX resources.
732fdf0e
GC		 1622 * @priv: private structure
1623 * Description: according to which descriptor can be used (extend or basic)
5bacd778
LC		 1624 * this function allocates the resources for TX and RX paths. In case of
1625 * reception, for example, it pre-allocated the RX socket buffer in order to
1626 * allow zero-copy mechanism.
732fdf0e 1627 */
71fedb01 1628static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv)
09f8d696 1629{
54139cf3 1630 u32 rx_count = priv->plat->rx_queues_to_use;
09f8d696 1631 int ret = -ENOMEM;
54139cf3 1632 u32 queue;
09f8d696 1633
54139cf3
JP		 1634 /* RX queues buffers and DMA */
1635 for (queue = 0; queue < rx_count; queue++) {
1636 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
2af6106a 1637 struct page_pool_params pp_params = { 0 };
4f28bd95 1638 unsigned int num_pages;
09f8d696 1639
54139cf3
JP		 1640 rx_q->queue_index = queue;
1641 rx_q->priv_data = priv;
5bacd778 1642
2af6106a
JA		 1643 pp_params.flags = PP_FLAG_DMA_MAP;
1644 pp_params.pool_size = DMA_RX_SIZE;
4f28bd95
TR		 1645 num_pages = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE);
1646 pp_params.order = ilog2(num_pages);
2af6106a
JA		 1647 pp_params.nid = dev_to_node(priv->device);
1648 pp_params.dev = priv->device;
1649 pp_params.dma_dir = DMA_FROM_DEVICE;
1650
1651 rx_q->page_pool = page_pool_create(&pp_params);
1652 if (IS_ERR(rx_q->page_pool)) {
1653 ret = PTR_ERR(rx_q->page_pool);
1654 rx_q->page_pool = NULL;
63c3aa6b 1655 goto err_dma;
2af6106a 1656 }
71fedb01 1657
ec5e5ce1
JA		 1658 rx_q->buf_pool = kcalloc(DMA_RX_SIZE, sizeof(*rx_q->buf_pool),
1659 GFP_KERNEL);
2af6106a 1660 if (!rx_q->buf_pool)
71fedb01 1661 goto err_dma;
54139cf3
JP		 1662
1663 if (priv->extend_desc) {
750afb08
LC		 1664 rx_q->dma_erx = dma_alloc_coherent(priv->device,
1665 DMA_RX_SIZE * sizeof(struct dma_extended_desc),
1666 &rx_q->dma_rx_phy,
1667 GFP_KERNEL);
54139cf3
JP		 1668 if (!rx_q->dma_erx)
1669 goto err_dma;
1670
1671 } else {
750afb08
LC		 1672 rx_q->dma_rx = dma_alloc_coherent(priv->device,
1673 DMA_RX_SIZE * sizeof(struct dma_desc),
1674 &rx_q->dma_rx_phy,
1675 GFP_KERNEL);
54139cf3
JP		 1676 if (!rx_q->dma_rx)
1677 goto err_dma;
1678 }
71fedb01
JP		 1679 }
1680
1681 return 0;
1682
1683err_dma:
54139cf3
JP		 1684 free_dma_rx_desc_resources(priv);
1685
71fedb01
JP		 1686 return ret;
1687}
1688
1689/**
1690 * alloc_dma_tx_desc_resources - alloc TX resources.
1691 * @priv: private structure
1692 * Description: according to which descriptor can be used (extend or basic)
1693 * this function allocates the resources for TX and RX paths. In case of
1694 * reception, for example, it pre-allocated the RX socket buffer in order to
1695 * allow zero-copy mechanism.
1696 */
1697static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv)
1698{
ce736788 1699 u32 tx_count = priv->plat->tx_queues_to_use;
71fedb01 1700 int ret = -ENOMEM;
ce736788 1701 u32 queue;
71fedb01 1702
ce736788
JP		 1703 /* TX queues buffers and DMA */
1704 for (queue = 0; queue < tx_count; queue++) {
1705 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
579a25a8
JA		 1706 size_t size;
1707 void *addr;
5bacd778 1708
ce736788
JP		 1709 tx_q->queue_index = queue;
1710 tx_q->priv_data = priv;
5bacd778 1711
ec5e5ce1
JA		 1712 tx_q->tx_skbuff_dma = kcalloc(DMA_TX_SIZE,
1713 sizeof(*tx_q->tx_skbuff_dma),
1714 GFP_KERNEL);
ce736788 1715 if (!tx_q->tx_skbuff_dma)
62242260 1716 goto err_dma;
ce736788 1717
ec5e5ce1
JA		 1718 tx_q->tx_skbuff = kcalloc(DMA_TX_SIZE,
1719 sizeof(struct sk_buff *),
1720 GFP_KERNEL);
ce736788 1721 if (!tx_q->tx_skbuff)
62242260 1722 goto err_dma;
ce736788 1723
579a25a8
JA		 1724 if (priv->extend_desc)
1725 size = sizeof(struct dma_extended_desc);
1726 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1727 size = sizeof(struct dma_edesc);
1728 else
1729 size = sizeof(struct dma_desc);
1730
1731 size *= DMA_TX_SIZE;
1732
1733 addr = dma_alloc_coherent(priv->device, size,
1734 &tx_q->dma_tx_phy, GFP_KERNEL);
1735 if (!addr)
1736 goto err_dma;
1737
1738 if (priv->extend_desc)
1739 tx_q->dma_etx = addr;
1740 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1741 tx_q->dma_entx = addr;
1742 else
1743 tx_q->dma_tx = addr;
09f8d696
SK		 1744 }
1745
1746 return 0;
1747
62242260 1748err_dma:
ce736788 1749 free_dma_tx_desc_resources(priv);
09f8d696
SK		 1750 return ret;
1751}
1752
71fedb01
JP		 1753/**
1754 * alloc_dma_desc_resources - alloc TX/RX resources.
1755 * @priv: private structure
1756 * Description: according to which descriptor can be used (extend or basic)
1757 * this function allocates the resources for TX and RX paths. In case of
1758 * reception, for example, it pre-allocated the RX socket buffer in order to
1759 * allow zero-copy mechanism.
1760 */
1761static int alloc_dma_desc_resources(struct stmmac_priv *priv)
1762{
54139cf3 1763 /* RX Allocation */
71fedb01
JP		 1764 int ret = alloc_dma_rx_desc_resources(priv);
1765
1766 if (ret)
1767 return ret;
1768
1769 ret = alloc_dma_tx_desc_resources(priv);
1770
1771 return ret;
1772}
1773
71fedb01
JP		 1774/**
1775 * free_dma_desc_resources - free dma desc resources
1776 * @priv: private structure
1777 */
1778static void free_dma_desc_resources(struct stmmac_priv *priv)
1779{
1780 /* Release the DMA RX socket buffers */
1781 free_dma_rx_desc_resources(priv);
1782
1783 /* Release the DMA TX socket buffers */
1784 free_dma_tx_desc_resources(priv);
1785}
1786
9eb12474 1787/**
1788 * stmmac_mac_enable_rx_queues - Enable MAC rx queues
1789 * @priv: driver private structure
1790 * Description: It is used for enabling the rx queues in the MAC
1791 */
1792static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
1793{
4f6046f5
JP		 1794 u32 rx_queues_count = priv->plat->rx_queues_to_use;
1795 int queue;
1796 u8 mode;
9eb12474 1797
4f6046f5
JP		 1798 for (queue = 0; queue < rx_queues_count; queue++) {
1799 mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
c10d4c82 1800 stmmac_rx_queue_enable(priv, priv->hw, mode, queue);
4f6046f5 1801 }
9eb12474 1802}
1803
ae4f0d46
JP		 1804/**
1805 * stmmac_start_rx_dma - start RX DMA channel
1806 * @priv: driver private structure
1807 * @chan: RX channel index
1808 * Description:
1809 * This starts a RX DMA channel
1810 */
1811static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
1812{
1813 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
a4e887fa 1814 stmmac_start_rx(priv, priv->ioaddr, chan);
ae4f0d46
JP		 1815}
1816
1817/**
1818 * stmmac_start_tx_dma - start TX DMA channel
1819 * @priv: driver private structure
1820 * @chan: TX channel index
1821 * Description:
1822 * This starts a TX DMA channel
1823 */
1824static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
1825{
1826 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
a4e887fa 1827 stmmac_start_tx(priv, priv->ioaddr, chan);
ae4f0d46
JP		 1828}
1829
1830/**
1831 * stmmac_stop_rx_dma - stop RX DMA channel
1832 * @priv: driver private structure
1833 * @chan: RX channel index
1834 * Description:
1835 * This stops a RX DMA channel
1836 */
1837static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
1838{
1839 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
a4e887fa 1840 stmmac_stop_rx(priv, priv->ioaddr, chan);
ae4f0d46
JP		 1841}
1842
1843/**
1844 * stmmac_stop_tx_dma - stop TX DMA channel
1845 * @priv: driver private structure
1846 * @chan: TX channel index
1847 * Description:
1848 * This stops a TX DMA channel
1849 */
1850static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
1851{
1852 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
a4e887fa 1853 stmmac_stop_tx(priv, priv->ioaddr, chan);
ae4f0d46
JP		 1854}
1855
1856/**
1857 * stmmac_start_all_dma - start all RX and TX DMA channels
1858 * @priv: driver private structure
1859 * Description:
1860 * This starts all the RX and TX DMA channels
1861 */
1862static void stmmac_start_all_dma(struct stmmac_priv *priv)
1863{
1864 u32 rx_channels_count = priv->plat->rx_queues_to_use;
1865 u32 tx_channels_count = priv->plat->tx_queues_to_use;
1866 u32 chan = 0;
1867
1868 for (chan = 0; chan < rx_channels_count; chan++)
1869 stmmac_start_rx_dma(priv, chan);
1870
1871 for (chan = 0; chan < tx_channels_count; chan++)
1872 stmmac_start_tx_dma(priv, chan);
1873}
1874
1875/**
1876 * stmmac_stop_all_dma - stop all RX and TX DMA channels
1877 * @priv: driver private structure
1878 * Description:
1879 * This stops the RX and TX DMA channels
1880 */
1881static void stmmac_stop_all_dma(struct stmmac_priv *priv)
1882{
1883 u32 rx_channels_count = priv->plat->rx_queues_to_use;
1884 u32 tx_channels_count = priv->plat->tx_queues_to_use;
1885 u32 chan = 0;
1886
1887 for (chan = 0; chan < rx_channels_count; chan++)
1888 stmmac_stop_rx_dma(priv, chan);
1889
1890 for (chan = 0; chan < tx_channels_count; chan++)
1891 stmmac_stop_tx_dma(priv, chan);
1892}
1893
47dd7a54
GC		 1894/**
1895 * stmmac_dma_operation_mode - HW DMA operation mode
32ceabca 1896 * @priv: driver private structure
732fdf0e
GC		 1897 * Description: it is used for configuring the DMA operation mode register in
1898 * order to program the tx/rx DMA thresholds or Store-And-Forward mode.
47dd7a54
GC		 1899 */
1900static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
1901{
6deee222
JP		 1902 u32 rx_channels_count = priv->plat->rx_queues_to_use;
1903 u32 tx_channels_count = priv->plat->tx_queues_to_use;
f88203a2 1904 int rxfifosz = priv->plat->rx_fifo_size;
52a76235 1905 int txfifosz = priv->plat->tx_fifo_size;
6deee222
JP		 1906 u32 txmode = 0;
1907 u32 rxmode = 0;
1908 u32 chan = 0;
a0daae13 1909 u8 qmode = 0;
f88203a2 1910
11fbf811
TR		 1911 if (rxfifosz == 0)
1912 rxfifosz = priv->dma_cap.rx_fifo_size;
52a76235
JA		 1913 if (txfifosz == 0)
1914 txfifosz = priv->dma_cap.tx_fifo_size;
1915
1916 /* Adjust for real per queue fifo size */
1917 rxfifosz /= rx_channels_count;
1918 txfifosz /= tx_channels_count;
11fbf811 1919
6deee222
JP		 1920 if (priv->plat->force_thresh_dma_mode) {
1921 txmode = tc;
1922 rxmode = tc;
1923 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
61b8013a
SK		 1924 /*
1925 * In case of GMAC, SF mode can be enabled
1926 * to perform the TX COE in HW. This depends on:
ebbb293f
GC		 1927 * 1) TX COE if actually supported
1928 * 2) There is no bugged Jumbo frame support
1929 * that needs to not insert csum in the TDES.
1930 */
6deee222
JP		 1931 txmode = SF_DMA_MODE;
1932 rxmode = SF_DMA_MODE;
b2dec116 1933 priv->xstats.threshold = SF_DMA_MODE;
6deee222
JP		 1934 } else {
1935 txmode = tc;
1936 rxmode = SF_DMA_MODE;
1937 }
1938
1939 /* configure all channels */
ab0204e3
JA		 1940 for (chan = 0; chan < rx_channels_count; chan++) {
1941 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
a0daae13 1942
ab0204e3
JA		 1943 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan,
1944 rxfifosz, qmode);
4205c88e
JA		 1945 stmmac_set_dma_bfsize(priv, priv->ioaddr, priv->dma_buf_sz,
1946 chan);
ab0204e3 1947 }
a0daae13 1948
ab0204e3
JA		 1949 for (chan = 0; chan < tx_channels_count; chan++) {
1950 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
6deee222 1951
ab0204e3
JA		 1952 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan,
1953 txfifosz, qmode);
6deee222 1954 }
47dd7a54
GC		 1955}
1956
47dd7a54 1957/**
732fdf0e 1958 * stmmac_tx_clean - to manage the transmission completion
32ceabca 1959 * @priv: driver private structure
ce736788 1960 * @queue: TX queue index
732fdf0e 1961 * Description: it reclaims the transmit resources after transmission completes.
47dd7a54 1962 */
8fce3331 1963static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue)
47dd7a54 1964{
ce736788 1965 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
38979574 1966 unsigned int bytes_compl = 0, pkts_compl = 0;
8fce3331 1967 unsigned int entry, count = 0;
47dd7a54 1968
8fce3331 1969 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue));
a9097a96 1970
9125cdd1
GC		 1971 priv->xstats.tx_clean++;
1972
8d5f4b07 1973 entry = tx_q->dirty_tx;
8fce3331 1974 while ((entry != tx_q->cur_tx) && (count < budget)) {
ce736788 1975 struct sk_buff *skb = tx_q->tx_skbuff[entry];
c24602ef 1976 struct dma_desc *p;
c363b658 1977 int status;
c24602ef
GC		 1978
1979 if (priv->extend_desc)
ce736788 1980 p = (struct dma_desc *)(tx_q->dma_etx + entry);
579a25a8
JA		 1981 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1982 p = &tx_q->dma_entx[entry].basic;
c24602ef 1983 else
ce736788 1984 p = tx_q->dma_tx + entry;
47dd7a54 1985
42de047d
JA		 1986 status = stmmac_tx_status(priv, &priv->dev->stats,
1987 &priv->xstats, p, priv->ioaddr);
c363b658
FG		 1988 /* Check if the descriptor is owned by the DMA */
1989 if (unlikely(status & tx_dma_own))
1990 break;
1991
8fce3331
JA		 1992 count++;
1993
a6b25da5
NC		 1994 /* Make sure descriptor fields are read after reading
1995 * the own bit.
1996 */
1997 dma_rmb();
1998
c363b658
FG		 1999 /* Just consider the last segment and ...*/
2000 if (likely(!(status & tx_not_ls))) {
2001 /* ... verify the status error condition */
2002 if (unlikely(status & tx_err)) {
2003 priv->dev->stats.tx_errors++;
2004 } else {
47dd7a54
GC		 2005 priv->dev->stats.tx_packets++;
2006 priv->xstats.tx_pkt_n++;
c363b658 2007 }
ba1ffd74 2008 stmmac_get_tx_hwtstamp(priv, p, skb);
47dd7a54 2009 }
47dd7a54 2010
ce736788
JP		 2011 if (likely(tx_q->tx_skbuff_dma[entry].buf)) {
2012 if (tx_q->tx_skbuff_dma[entry].map_as_page)
362b37be 2013 dma_unmap_page(priv->device,
ce736788
JP		 2014 tx_q->tx_skbuff_dma[entry].buf,
2015 tx_q->tx_skbuff_dma[entry].len,
362b37be
GC		 2016 DMA_TO_DEVICE);
2017 else
2018 dma_unmap_single(priv->device,
ce736788
JP		 2019 tx_q->tx_skbuff_dma[entry].buf,
2020 tx_q->tx_skbuff_dma[entry].len,
362b37be 2021 DMA_TO_DEVICE);
ce736788
JP		 2022 tx_q->tx_skbuff_dma[entry].buf = 0;
2023 tx_q->tx_skbuff_dma[entry].len = 0;
2024 tx_q->tx_skbuff_dma[entry].map_as_page = false;
cf32deec 2025 }
f748be53 2026
2c520b1c 2027 stmmac_clean_desc3(priv, tx_q, p);
f748be53 2028
ce736788
JP		 2029 tx_q->tx_skbuff_dma[entry].last_segment = false;
2030 tx_q->tx_skbuff_dma[entry].is_jumbo = false;
47dd7a54
GC		 2031
2032 if (likely(skb != NULL)) {
38979574
BG		 2033 pkts_compl++;
2034 bytes_compl += skb->len;
7c565c33 2035 dev_consume_skb_any(skb);
ce736788 2036 tx_q->tx_skbuff[entry] = NULL;
47dd7a54
GC		 2037 }
2038
42de047d 2039 stmmac_release_tx_desc(priv, p, priv->mode);
47dd7a54 2040
e3ad57c9 2041 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
47dd7a54 2042 }
ce736788 2043 tx_q->dirty_tx = entry;
38979574 2044
c22a3f48
JP		 2045 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
2046 pkts_compl, bytes_compl);
2047
2048 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
2049 queue))) &&
2050 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) {
38979574 2051
739c8e14
LS		 2052 netif_dbg(priv, tx_done, priv->dev,
2053 "%s: restart transmit\n", __func__);
c22a3f48 2054 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
47dd7a54 2055 }
d765955d
GC		 2056
2057 if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
2058 stmmac_enable_eee_mode(priv);
f5351ef7 2059 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
d765955d 2060 }
8fce3331 2061
4ccb4585
JA		 2062 /* We still have pending packets, let's call for a new scheduling */
2063 if (tx_q->dirty_tx != tx_q->cur_tx)
3755b21b 2064 mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer));
4ccb4585 2065
8fce3331
JA		 2066 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue));
2067
2068 return count;
47dd7a54
GC		 2069}
2070
47dd7a54 2071/**
732fdf0e 2072 * stmmac_tx_err - to manage the tx error
32ceabca 2073 * @priv: driver private structure
5bacd778 2074 * @chan: channel index
47dd7a54 2075 * Description: it cleans the descriptors and restarts the transmission
732fdf0e 2076 * in case of transmission errors.
47dd7a54 2077 */
5bacd778 2078static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
47dd7a54 2079{
ce736788 2080 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
ce736788 2081
c22a3f48 2082 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));
47dd7a54 2083
ae4f0d46 2084 stmmac_stop_tx_dma(priv, chan);
ce736788 2085 dma_free_tx_skbufs(priv, chan);
579a25a8 2086 stmmac_clear_tx_descriptors(priv, chan);
ce736788
JP		 2087 tx_q->dirty_tx = 0;
2088 tx_q->cur_tx = 0;
8d212a9e 2089 tx_q->mss = 0;
c22a3f48 2090 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan));
f421031e
JK		 2091 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2092 tx_q->dma_tx_phy, chan);
ae4f0d46 2093 stmmac_start_tx_dma(priv, chan);
47dd7a54
GC		 2094
2095 priv->dev->stats.tx_errors++;
c22a3f48 2096 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));
47dd7a54
GC		 2097}
2098
6deee222
JP		 2099/**
2100 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel
2101 * @priv: driver private structure
2102 * @txmode: TX operating mode
2103 * @rxmode: RX operating mode
2104 * @chan: channel index
2105 * Description: it is used for configuring of the DMA operation mode in
2106 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
2107 * mode.
2108 */
2109static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
2110 u32 rxmode, u32 chan)
2111{
a0daae13
JA		 2112 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
2113 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
52a76235
JA		 2114 u32 rx_channels_count = priv->plat->rx_queues_to_use;
2115 u32 tx_channels_count = priv->plat->tx_queues_to_use;
6deee222 2116 int rxfifosz = priv->plat->rx_fifo_size;
52a76235 2117 int txfifosz = priv->plat->tx_fifo_size;
6deee222
JP		 2118
2119 if (rxfifosz == 0)
2120 rxfifosz = priv->dma_cap.rx_fifo_size;
52a76235
JA		 2121 if (txfifosz == 0)
2122 txfifosz = priv->dma_cap.tx_fifo_size;
2123
2124 /* Adjust for real per queue fifo size */
2125 rxfifosz /= rx_channels_count;
2126 txfifosz /= tx_channels_count;
6deee222 2127
ab0204e3
JA		 2128 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode);
2129 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode);
6deee222
JP		 2130}
2131
8bf993a5
JA		 2132static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv)
2133{
63a550fc 2134 int ret;
8bf993a5 2135
c10d4c82
JA		 2136 ret = stmmac_safety_feat_irq_status(priv, priv->dev,
2137 priv->ioaddr, priv->dma_cap.asp, &priv->sstats);
2138 if (ret && (ret != -EINVAL)) {
8bf993a5 2139 stmmac_global_err(priv);
c10d4c82
JA		 2140 return true;
2141 }
2142
2143 return false;
8bf993a5
JA		 2144}
2145
8fce3331
JA		 2146static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan)
2147{
2148 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr,
2149 &priv->xstats, chan);
2150 struct stmmac_channel *ch = &priv->channel[chan];
021bd5e3 2151 unsigned long flags;
8fce3331 2152
4ccb4585 2153 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) {
3ba07deb 2154 if (napi_schedule_prep(&ch->rx_napi)) {
021bd5e3
JA		 2155 spin_lock_irqsave(&ch->lock, flags);
2156 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0);
2157 spin_unlock_irqrestore(&ch->lock, flags);
3ba07deb 2158 __napi_schedule_irqoff(&ch->rx_napi);
3ba07deb 2159 }
8fce3331
JA		 2160 }
2161
021bd5e3
JA		 2162 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) {
2163 if (napi_schedule_prep(&ch->tx_napi)) {
2164 spin_lock_irqsave(&ch->lock, flags);
2165 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1);
2166 spin_unlock_irqrestore(&ch->lock, flags);
2167 __napi_schedule_irqoff(&ch->tx_napi);
2168 }
2169 }
8fce3331
JA		 2170
2171 return status;
2172}
2173
32ceabca 2174/**
732fdf0e 2175 * stmmac_dma_interrupt - DMA ISR
32ceabca
GC		 2176 * @priv: driver private structure
2177 * Description: this is the DMA ISR. It is called by the main ISR.
732fdf0e
GC		 2178 * It calls the dwmac dma routine and schedule poll method in case of some
2179 * work can be done.
32ceabca 2180 */
aec7ff27
GC		 2181static void stmmac_dma_interrupt(struct stmmac_priv *priv)
2182{
d62a107a 2183 u32 tx_channel_count = priv->plat->tx_queues_to_use;
5a6a0445
NC		 2184 u32 rx_channel_count = priv->plat->rx_queues_to_use;
2185 u32 channels_to_check = tx_channel_count > rx_channel_count ?
2186 tx_channel_count : rx_channel_count;
d62a107a 2187 u32 chan;
8ac60ffb
KC		 2188 int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)];
2189
2190 /* Make sure we never check beyond our status buffer. */
2191 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status)))
2192 channels_to_check = ARRAY_SIZE(status);
5a6a0445 2193
5a6a0445 2194 for (chan = 0; chan < channels_to_check; chan++)
8fce3331 2195 status[chan] = stmmac_napi_check(priv, chan);
6deee222 2196
5a6a0445
NC		 2197 for (chan = 0; chan < tx_channel_count; chan++) {
2198 if (unlikely(status[chan] & tx_hard_error_bump_tc)) {
d62a107a
JP		 2199 /* Try to bump up the dma threshold on this failure */
2200 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
2201 (tc <= 256)) {
2202 tc += 64;
2203 if (priv->plat->force_thresh_dma_mode)
2204 stmmac_set_dma_operation_mode(priv,
2205 tc,
2206 tc,
2207 chan);
2208 else
2209 stmmac_set_dma_operation_mode(priv,
2210 tc,
2211 SF_DMA_MODE,
2212 chan);
2213 priv->xstats.threshold = tc;
2214 }
5a6a0445 2215 } else if (unlikely(status[chan] == tx_hard_error)) {
d62a107a 2216 stmmac_tx_err(priv, chan);
47dd7a54 2217 }
d62a107a 2218 }
47dd7a54
GC		 2219}
2220
32ceabca
GC		 2221/**
2222 * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
2223 * @priv: driver private structure
2224 * Description: this masks the MMC irq, in fact, the counters are managed in SW.
2225 */
1c901a46
GC		 2226static void stmmac_mmc_setup(struct stmmac_priv *priv)
2227{
2228 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
36ff7c1e 2229 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
1c901a46 2230
3b1dd2c5 2231 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr);
4f795b25
GC		 2232
2233 if (priv->dma_cap.rmon) {
3b1dd2c5 2234 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode);
4f795b25
GC		 2235 memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
2236 } else
38ddc59d 2237 netdev_info(priv->dev, "No MAC Management Counters available\n");
1c901a46
GC		 2238}
2239
19e30c14 2240/**
732fdf0e 2241 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
32ceabca 2242 * @priv: driver private structure
19e30c14
GC		 2243 * Description:
2244 * new GMAC chip generations have a new register to indicate the
2245 * presence of the optional feature/functions.
2246 * This can be also used to override the value passed through the
2247 * platform and necessary for old MAC10/100 and GMAC chips.
e7434821
GC		 2248 */
2249static int stmmac_get_hw_features(struct stmmac_priv *priv)
2250{
a4e887fa 2251 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0;
e7434821
GC		 2252}
2253
32ceabca 2254/**
732fdf0e 2255 * stmmac_check_ether_addr - check if the MAC addr is valid
32ceabca
GC		 2256 * @priv: driver private structure
2257 * Description:
2258 * it is to verify if the MAC address is valid, in case of failures it
2259 * generates a random MAC address
2260 */
bfab27a1
GC		 2261static void stmmac_check_ether_addr(struct stmmac_priv *priv)
2262{
bfab27a1 2263 if (!is_valid_ether_addr(priv->dev->dev_addr)) {
c10d4c82 2264 stmmac_get_umac_addr(priv, priv->hw, priv->dev->dev_addr, 0);
ceb69499 2265 if (!is_valid_ether_addr(priv->dev->dev_addr))
f2cedb63 2266 eth_hw_addr_random(priv->dev);
af649352
JZ		 2267 dev_info(priv->device, "device MAC address %pM\n",
2268 priv->dev->dev_addr);
bfab27a1 2269 }
bfab27a1
GC		 2270}
2271
32ceabca 2272/**
732fdf0e 2273 * stmmac_init_dma_engine - DMA init.
32ceabca
GC		 2274 * @priv: driver private structure
2275 * Description:
2276 * It inits the DMA invoking the specific MAC/GMAC callback.
2277 * Some DMA parameters can be passed from the platform;
2278 * in case of these are not passed a default is kept for the MAC or GMAC.
2279 */
0f1f88a8
GC		 2280static int stmmac_init_dma_engine(struct stmmac_priv *priv)
2281{
47f2a9ce
JP		 2282 u32 rx_channels_count = priv->plat->rx_queues_to_use;
2283 u32 tx_channels_count = priv->plat->tx_queues_to_use;
24aaed0c 2284 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
54139cf3 2285 struct stmmac_rx_queue *rx_q;
ce736788 2286 struct stmmac_tx_queue *tx_q;
47f2a9ce 2287 u32 chan = 0;
c24602ef 2288 int atds = 0;
495db273 2289 int ret = 0;
0f1f88a8 2290
a332e2fa
NC		 2291 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
2292 dev_err(priv->device, "Invalid DMA configuration\n");
89ab75bf 2293 return -EINVAL;
0f1f88a8
GC		 2294 }
2295
c24602ef
GC		 2296 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
2297 atds = 1;
2298
a4e887fa 2299 ret = stmmac_reset(priv, priv->ioaddr);
495db273
GC		 2300 if (ret) {
2301 dev_err(priv->device, "Failed to reset the dma\n");
2302 return ret;
2303 }
2304
7d9e6c5a
JA		 2305 /* DMA Configuration */
2306 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds);
2307
2308 if (priv->plat->axi)
2309 stmmac_axi(priv, priv->ioaddr, priv->plat->axi);
2310
af8f3fb7
WV		 2311 /* DMA CSR Channel configuration */
2312 for (chan = 0; chan < dma_csr_ch; chan++)
2313 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
2314
24aaed0c
JA		 2315 /* DMA RX Channel Configuration */
2316 for (chan = 0; chan < rx_channels_count; chan++) {
2317 rx_q = &priv->rx_queue[chan];
47f2a9ce 2318
24aaed0c
JA		 2319 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2320 rx_q->dma_rx_phy, chan);
54139cf3 2321
24aaed0c
JA		 2322 rx_q->rx_tail_addr = rx_q->dma_rx_phy +
2323 (DMA_RX_SIZE * sizeof(struct dma_desc));
2324 stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
2325 rx_q->rx_tail_addr, chan);
2326 }
47f2a9ce 2327
24aaed0c
JA		 2328 /* DMA TX Channel Configuration */
2329 for (chan = 0; chan < tx_channels_count; chan++) {
2330 tx_q = &priv->tx_queue[chan];
47f2a9ce 2331
24aaed0c
JA		 2332 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2333 tx_q->dma_tx_phy, chan);
ce736788 2334
0431100b 2335 tx_q->tx_tail_addr = tx_q->dma_tx_phy;
24aaed0c
JA		 2336 stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
2337 tx_q->tx_tail_addr, chan);
2338 }
47f2a9ce 2339
495db273 2340 return ret;
0f1f88a8
GC		 2341}
2342
8fce3331
JA		 2343static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue)
2344{
2345 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2346
2347 mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer));
2348}
2349
9125cdd1 2350/**
732fdf0e 2351 * stmmac_tx_timer - mitigation sw timer for tx.
9125cdd1
GC		 2352 * @data: data pointer
2353 * Description:
2354 * This is the timer handler to directly invoke the stmmac_tx_clean.
2355 */
e99e88a9 2356static void stmmac_tx_timer(struct timer_list *t)
9125cdd1 2357{
8fce3331
JA		 2358 struct stmmac_tx_queue *tx_q = from_timer(tx_q, t, txtimer);
2359 struct stmmac_priv *priv = tx_q->priv_data;
2360 struct stmmac_channel *ch;
2361
2362 ch = &priv->channel[tx_q->queue_index];
9125cdd1 2363
021bd5e3
JA		 2364 if (likely(napi_schedule_prep(&ch->tx_napi))) {
2365 unsigned long flags;
2366
2367 spin_lock_irqsave(&ch->lock, flags);
2368 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1);
2369 spin_unlock_irqrestore(&ch->lock, flags);
4ccb4585 2370 __napi_schedule(&ch->tx_napi);
021bd5e3 2371 }
9125cdd1
GC		 2372}
2373
2374/**
d429b66e 2375 * stmmac_init_coalesce - init mitigation options.
32ceabca 2376 * @priv: driver private structure
9125cdd1 2377 * Description:
d429b66e 2378 * This inits the coalesce parameters: i.e. timer rate,
9125cdd1
GC		 2379 * timer handler and default threshold used for enabling the
2380 * interrupt on completion bit.
2381 */
d429b66e 2382static void stmmac_init_coalesce(struct stmmac_priv *priv)
9125cdd1 2383{
8fce3331
JA		 2384 u32 tx_channel_count = priv->plat->tx_queues_to_use;
2385 u32 chan;
2386
9125cdd1
GC		 2387 priv->tx_coal_frames = STMMAC_TX_FRAMES;
2388 priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
d429b66e 2389 priv->rx_coal_frames = STMMAC_RX_FRAMES;
8fce3331
JA		 2390
2391 for (chan = 0; chan < tx_channel_count; chan++) {
2392 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2393
2394 timer_setup(&tx_q->txtimer, stmmac_tx_timer, 0);
2395 }
9125cdd1
GC		 2396}
2397
4854ab99
JP		 2398static void stmmac_set_rings_length(struct stmmac_priv *priv)
2399{
2400 u32 rx_channels_count = priv->plat->rx_queues_to_use;
2401 u32 tx_channels_count = priv->plat->tx_queues_to_use;
2402 u32 chan;
2403
2404 /* set TX ring length */
a4e887fa
JA		 2405 for (chan = 0; chan < tx_channels_count; chan++)
2406 stmmac_set_tx_ring_len(priv, priv->ioaddr,
2407 (DMA_TX_SIZE - 1), chan);
4854ab99
JP		 2408
2409 /* set RX ring length */
a4e887fa
JA		 2410 for (chan = 0; chan < rx_channels_count; chan++)
2411 stmmac_set_rx_ring_len(priv, priv->ioaddr,
2412 (DMA_RX_SIZE - 1), chan);
4854ab99
JP		 2413}
2414
6a3a7193
JP		 2415/**
2416 * stmmac_set_tx_queue_weight - Set TX queue weight
2417 * @priv: driver private structure
2418 * Description: It is used for setting TX queues weight
2419 */
2420static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
2421{
2422 u32 tx_queues_count = priv->plat->tx_queues_to_use;
2423 u32 weight;
2424 u32 queue;
2425
2426 for (queue = 0; queue < tx_queues_count; queue++) {
2427 weight = priv->plat->tx_queues_cfg[queue].weight;
c10d4c82 2428 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue);
6a3a7193
JP		 2429 }
2430}
2431
19d91873
JP		 2432/**
2433 * stmmac_configure_cbs - Configure CBS in TX queue
2434 * @priv: driver private structure
2435 * Description: It is used for configuring CBS in AVB TX queues
2436 */
2437static void stmmac_configure_cbs(struct stmmac_priv *priv)
2438{
2439 u32 tx_queues_count = priv->plat->tx_queues_to_use;
2440 u32 mode_to_use;
2441 u32 queue;
2442
44781fef
JP		 2443 /* queue 0 is reserved for legacy traffic */
2444 for (queue = 1; queue < tx_queues_count; queue++) {
19d91873
JP		 2445 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
2446 if (mode_to_use == MTL_QUEUE_DCB)
2447 continue;
2448
c10d4c82 2449 stmmac_config_cbs(priv, priv->hw,
19d91873
JP		 2450 priv->plat->tx_queues_cfg[queue].send_slope,
2451 priv->plat->tx_queues_cfg[queue].idle_slope,
2452 priv->plat->tx_queues_cfg[queue].high_credit,
2453 priv->plat->tx_queues_cfg[queue].low_credit,
2454 queue);
2455 }
2456}
2457
d43042f4
JP		 2458/**
2459 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
2460 * @priv: driver private structure
2461 * Description: It is used for mapping RX queues to RX dma channels
2462 */
2463static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
2464{
2465 u32 rx_queues_count = priv->plat->rx_queues_to_use;
2466 u32 queue;
2467 u32 chan;
2468
2469 for (queue = 0; queue < rx_queues_count; queue++) {
2470 chan = priv->plat->rx_queues_cfg[queue].chan;
c10d4c82 2471 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan);
d43042f4
JP		 2472 }
2473}
2474
a8f5102a
JP		 2475/**
2476 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
2477 * @priv: driver private structure
2478 * Description: It is used for configuring the RX Queue Priority
2479 */
2480static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
2481{
2482 u32 rx_queues_count = priv->plat->rx_queues_to_use;
2483 u32 queue;
2484 u32 prio;
2485
2486 for (queue = 0; queue < rx_queues_count; queue++) {
2487 if (!priv->plat->rx_queues_cfg[queue].use_prio)
2488 continue;
2489
2490 prio = priv->plat->rx_queues_cfg[queue].prio;
c10d4c82 2491 stmmac_rx_queue_prio(priv, priv->hw, prio, queue);
a8f5102a
JP		 2492 }
2493}
2494
2495/**
2496 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
2497 * @priv: driver private structure
2498 * Description: It is used for configuring the TX Queue Priority
2499 */
2500static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
2501{
2502 u32 tx_queues_count = priv->plat->tx_queues_to_use;
2503 u32 queue;
2504 u32 prio;
2505
2506 for (queue = 0; queue < tx_queues_count; queue++) {
2507 if (!priv->plat->tx_queues_cfg[queue].use_prio)
2508 continue;
2509
2510 prio = priv->plat->tx_queues_cfg[queue].prio;
c10d4c82 2511 stmmac_tx_queue_prio(priv, priv->hw, prio, queue);
a8f5102a
JP		 2512 }
2513}
2514
abe80fdc
JP		 2515/**
2516 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
2517 * @priv: driver private structure
2518 * Description: It is used for configuring the RX queue routing
2519 */
2520static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
2521{
2522 u32 rx_queues_count = priv->plat->rx_queues_to_use;
2523 u32 queue;
2524 u8 packet;
2525
2526 for (queue = 0; queue < rx_queues_count; queue++) {
2527 /* no specific packet type routing specified for the queue */
2528 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
2529 continue;
2530
2531 packet = priv->plat->rx_queues_cfg[queue].pkt_route;
c10d4c82 2532 stmmac_rx_queue_routing(priv, priv->hw, packet, queue);
abe80fdc
JP		 2533 }
2534}
2535
76067459
JA		 2536static void stmmac_mac_config_rss(struct stmmac_priv *priv)
2537{
2538 if (!priv->dma_cap.rssen || !priv->plat->rss_en) {
2539 priv->rss.enable = false;
2540 return;
2541 }
2542
2543 if (priv->dev->features & NETIF_F_RXHASH)
2544 priv->rss.enable = true;
2545 else
2546 priv->rss.enable = false;
2547
2548 stmmac_rss_configure(priv, priv->hw, &priv->rss,
2549 priv->plat->rx_queues_to_use);
2550}
2551
d0a9c9f9
JP		 2552/**
2553 * stmmac_mtl_configuration - Configure MTL
2554 * @priv: driver private structure
2555 * Description: It is used for configurring MTL
2556 */
2557static void stmmac_mtl_configuration(struct stmmac_priv *priv)
2558{
2559 u32 rx_queues_count = priv->plat->rx_queues_to_use;
2560 u32 tx_queues_count = priv->plat->tx_queues_to_use;
2561
c10d4c82 2562 if (tx_queues_count > 1)
6a3a7193
JP		 2563 stmmac_set_tx_queue_weight(priv);
2564
d0a9c9f9 2565 /* Configure MTL RX algorithms */
c10d4c82
JA		 2566 if (rx_queues_count > 1)
2567 stmmac_prog_mtl_rx_algorithms(priv, priv->hw,
2568 priv->plat->rx_sched_algorithm);
d0a9c9f9
JP		 2569
2570 /* Configure MTL TX algorithms */
c10d4c82
JA		 2571 if (tx_queues_count > 1)
2572 stmmac_prog_mtl_tx_algorithms(priv, priv->hw,
2573 priv->plat->tx_sched_algorithm);
d0a9c9f9 2574
19d91873 2575 /* Configure CBS in AVB TX queues */
c10d4c82 2576 if (tx_queues_count > 1)
19d91873
JP		 2577 stmmac_configure_cbs(priv);
2578
d43042f4 2579 /* Map RX MTL to DMA channels */
c10d4c82 2580 stmmac_rx_queue_dma_chan_map(priv);
d43042f4 2581
d0a9c9f9 2582 /* Enable MAC RX Queues */
c10d4c82 2583 stmmac_mac_enable_rx_queues(priv);
6deee222 2584
a8f5102a 2585 /* Set RX priorities */
c10d4c82 2586 if (rx_queues_count > 1)
a8f5102a
JP		 2587 stmmac_mac_config_rx_queues_prio(priv);
2588
2589 /* Set TX priorities */
c10d4c82 2590 if (tx_queues_count > 1)
a8f5102a 2591 stmmac_mac_config_tx_queues_prio(priv);
abe80fdc
JP		 2592
2593 /* Set RX routing */
c10d4c82 2594 if (rx_queues_count > 1)
abe80fdc 2595 stmmac_mac_config_rx_queues_routing(priv);
76067459
JA		 2596
2597 /* Receive Side Scaling */
2598 if (rx_queues_count > 1)
2599 stmmac_mac_config_rss(priv);
d0a9c9f9
JP		 2600}
2601
8bf993a5
JA		 2602static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)
2603{
c10d4c82 2604 if (priv->dma_cap.asp) {
8bf993a5 2605 netdev_info(priv->dev, "Enabling Safety Features\n");
c10d4c82 2606 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp);
8bf993a5
JA		 2607 } else {
2608 netdev_info(priv->dev, "No Safety Features support found\n");
2609 }
2610}
2611
523f11b5 2612/**
732fdf0e 2613 * stmmac_hw_setup - setup mac in a usable state.
523f11b5
SK		 2614 * @dev : pointer to the device structure.
2615 * Description:
732fdf0e
GC		 2616 * this is the main function to setup the HW in a usable state because the
2617 * dma engine is reset, the core registers are configured (e.g. AXI,
2618 * Checksum features, timers). The DMA is ready to start receiving and
2619 * transmitting.
523f11b5
SK		 2620 * Return value:
2621 * 0 on success and an appropriate (-)ve integer as defined in errno.h
2622 * file on failure.
2623 */
fe131929 2624static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
523f11b5
SK		 2625{
2626 struct stmmac_priv *priv = netdev_priv(dev);
3c55d4d0 2627 u32 rx_cnt = priv->plat->rx_queues_to_use;
146617b8
JP		 2628 u32 tx_cnt = priv->plat->tx_queues_to_use;
2629 u32 chan;
523f11b5
SK		 2630 int ret;
2631
523f11b5
SK		 2632 /* DMA initialization and SW reset */
2633 ret = stmmac_init_dma_engine(priv);
2634 if (ret < 0) {
38ddc59d
LC		 2635 netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
2636 __func__);
523f11b5
SK		 2637 return ret;
2638 }
2639
2640 /* Copy the MAC addr into the HW */
c10d4c82 2641 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0);
523f11b5 2642
02e57b9d
GC		 2643 /* PS and related bits will be programmed according to the speed */
2644 if (priv->hw->pcs) {
2645 int speed = priv->plat->mac_port_sel_speed;
2646
2647 if ((speed == SPEED_10) || (speed == SPEED_100) ||
2648 (speed == SPEED_1000)) {
2649 priv->hw->ps = speed;
2650 } else {
2651 dev_warn(priv->device, "invalid port speed\n");
2652 priv->hw->ps = 0;
2653 }
2654 }
2655
523f11b5 2656 /* Initialize the MAC Core */
c10d4c82 2657 stmmac_core_init(priv, priv->hw, dev);
523f11b5 2658
d0a9c9f9 2659 /* Initialize MTL*/
63a550fc 2660 stmmac_mtl_configuration(priv);
9eb12474 2661
8bf993a5 2662 /* Initialize Safety Features */
63a550fc 2663 stmmac_safety_feat_configuration(priv);
8bf993a5 2664
c10d4c82 2665 ret = stmmac_rx_ipc(priv, priv->hw);
978aded4 2666 if (!ret) {
38ddc59d 2667 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
978aded4 2668 priv->plat->rx_coe = STMMAC_RX_COE_NONE;
d2afb5bd 2669 priv->hw->rx_csum = 0;
978aded4
GC		 2670 }
2671
523f11b5 2672 /* Enable the MAC Rx/Tx */
c10d4c82 2673 stmmac_mac_set(priv, priv->ioaddr, true);
523f11b5 2674
b4f0a661
JP		 2675 /* Set the HW DMA mode and the COE */
2676 stmmac_dma_operation_mode(priv);
2677
523f11b5
SK		 2678 stmmac_mmc_setup(priv);
2679
fe131929 2680 if (init_ptp) {
0ad2be79
TR		 2681 ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
2682 if (ret < 0)
2683 netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret);
2684
fe131929 2685 ret = stmmac_init_ptp(priv);
722eef28
HK		 2686 if (ret == -EOPNOTSUPP)
2687 netdev_warn(priv->dev, "PTP not supported by HW\n");
2688 else if (ret)
2689 netdev_warn(priv->dev, "PTP init failed\n");
fe131929 2690 }
523f11b5 2691
523f11b5
SK		 2692 priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS;
2693
a4e887fa 2694 if (priv->use_riwt) {
4e4337cc
JA		 2695 if (!priv->rx_riwt)
2696 priv->rx_riwt = DEF_DMA_RIWT;
2697
2698 ret = stmmac_rx_watchdog(priv, priv->ioaddr, priv->rx_riwt, rx_cnt);
523f11b5
SK		 2699 }
2700
c10d4c82 2701 if (priv->hw->pcs)
c9ad4c10 2702 stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0);
523f11b5 2703
4854ab99
JP		 2704 /* set TX and RX rings length */
2705 stmmac_set_rings_length(priv);
2706
f748be53 2707 /* Enable TSO */
146617b8
JP		 2708 if (priv->tso) {
2709 for (chan = 0; chan < tx_cnt; chan++)
a4e887fa 2710 stmmac_enable_tso(priv, priv->ioaddr, 1, chan);
146617b8 2711 }
f748be53 2712
67afd6d1
JA		 2713 /* Enable Split Header */
2714 if (priv->sph && priv->hw->rx_csum) {
2715 for (chan = 0; chan < rx_cnt; chan++)
2716 stmmac_enable_sph(priv, priv->ioaddr, 1, chan);
2717 }
2718
30d93227
JA		 2719 /* VLAN Tag Insertion */
2720 if (priv->dma_cap.vlins)
2721 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT);
2722
579a25a8
JA		 2723 /* TBS */
2724 for (chan = 0; chan < tx_cnt; chan++) {
2725 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2726 int enable = tx_q->tbs & STMMAC_TBS_AVAIL;
2727
2728 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan);
2729 }
2730
7d9e6c5a
JA		 2731 /* Start the ball rolling... */
2732 stmmac_start_all_dma(priv);
2733
523f11b5
SK		 2734 return 0;
2735}
2736
c66f6c37
TR		 2737static void stmmac_hw_teardown(struct net_device *dev)
2738{
2739 struct stmmac_priv *priv = netdev_priv(dev);
2740
2741 clk_disable_unprepare(priv->plat->clk_ptp_ref);
2742}
2743
47dd7a54
GC		 2744/**
2745 * stmmac_open - open entry point of the driver
2746 * @dev : pointer to the device structure.
2747 * Description:
2748 * This function is the open entry point of the driver.
2749 * Return value:
2750 * 0 on success and an appropriate (-)ve integer as defined in errno.h
2751 * file on failure.
2752 */
2753static int stmmac_open(struct net_device *dev)
2754{
2755 struct stmmac_priv *priv = netdev_priv(dev);
5d626c87 2756 int bfsize = 0;
8fce3331 2757 u32 chan;
47dd7a54
GC		 2758 int ret;
2759
a47b9e15 2760 if (priv->hw->pcs != STMMAC_PCS_TBI &&
f213bbe8
JA		 2761 priv->hw->pcs != STMMAC_PCS_RTBI &&
2762 priv->hw->xpcs == NULL) {
e58bb43f
GC		 2763 ret = stmmac_init_phy(dev);
2764 if (ret) {
38ddc59d
LC		 2765 netdev_err(priv->dev,
2766 "%s: Cannot attach to PHY (error: %d)\n",
2767 __func__, ret);
89df20d9 2768 return ret;
e58bb43f 2769 }
f66ffe28 2770 }
47dd7a54 2771
523f11b5
SK		 2772 /* Extra statistics */
2773 memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
2774 priv->xstats.threshold = tc;
2775
5d626c87
JA		 2776 bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu);
2777 if (bfsize < 0)
2778 bfsize = 0;
2779
2780 if (bfsize < BUF_SIZE_16KiB)
2781 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
2782
2783 priv->dma_buf_sz = bfsize;
2784 buf_sz = bfsize;
2785
22ad3838 2786 priv->rx_copybreak = STMMAC_RX_COPYBREAK;
56329137 2787
579a25a8
JA		 2788 /* Earlier check for TBS */
2789 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) {
2790 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2791 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en;
2792
2793 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0;
2794 if (stmmac_enable_tbs(priv, priv->ioaddr, tbs_en, chan))
2795 tx_q->tbs &= ~STMMAC_TBS_AVAIL;
2796 }
2797
5bacd778
LC		 2798 ret = alloc_dma_desc_resources(priv);
2799 if (ret < 0) {
2800 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
2801 __func__);
2802 goto dma_desc_error;
2803 }
2804
2805 ret = init_dma_desc_rings(dev, GFP_KERNEL);
2806 if (ret < 0) {
2807 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
2808 __func__);
2809 goto init_error;
2810 }
2811
fe131929 2812 ret = stmmac_hw_setup(dev, true);
56329137 2813 if (ret < 0) {
38ddc59d 2814 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
c9324d18 2815 goto init_error;
47dd7a54
GC		 2816 }
2817
d429b66e 2818 stmmac_init_coalesce(priv);
777da230 2819
74371272 2820 phylink_start(priv->phylink);
47dd7a54 2821
f66ffe28
GC		 2822 /* Request the IRQ lines */
2823 ret = request_irq(dev->irq, stmmac_interrupt,
ceb69499 2824 IRQF_SHARED, dev->name, dev);
f66ffe28 2825 if (unlikely(ret < 0)) {
38ddc59d
LC		 2826 netdev_err(priv->dev,
2827 "%s: ERROR: allocating the IRQ %d (error: %d)\n",
2828 __func__, dev->irq, ret);
6c1e5abe 2829 goto irq_error;
f66ffe28
GC		 2830 }
2831
7a13f8f5
FV		 2832 /* Request the Wake IRQ in case of another line is used for WoL */
2833 if (priv->wol_irq != dev->irq) {
2834 ret = request_irq(priv->wol_irq, stmmac_interrupt,
2835 IRQF_SHARED, dev->name, dev);
2836 if (unlikely(ret < 0)) {
38ddc59d
LC		 2837 netdev_err(priv->dev,
2838 "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
2839 __func__, priv->wol_irq, ret);
c9324d18 2840 goto wolirq_error;
7a13f8f5
FV		 2841 }
2842 }
2843
d765955d 2844 /* Request the IRQ lines */
d7ec8584 2845 if (priv->lpi_irq > 0) {
d765955d
GC		 2846 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED,
2847 dev->name, dev);
2848 if (unlikely(ret < 0)) {
38ddc59d
LC		 2849 netdev_err(priv->dev,
2850 "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
2851 __func__, priv->lpi_irq, ret);
c9324d18 2852 goto lpiirq_error;
d765955d
GC		 2853 }
2854 }
2855
c22a3f48
JP		 2856 stmmac_enable_all_queues(priv);
2857 stmmac_start_all_queues(priv);
f66ffe28 2858
47dd7a54 2859 return 0;
f66ffe28 2860
c9324d18 2861lpiirq_error:
d765955d
GC		 2862 if (priv->wol_irq != dev->irq)
2863 free_irq(priv->wol_irq, dev);
c9324d18 2864wolirq_error:
7a13f8f5 2865 free_irq(dev->irq, dev);
6c1e5abe 2866irq_error:
74371272 2867 phylink_stop(priv->phylink);
7a13f8f5 2868
8fce3331
JA		 2869 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
2870 del_timer_sync(&priv->tx_queue[chan].txtimer);
2871
c66f6c37 2872 stmmac_hw_teardown(dev);
c9324d18
GC		 2873init_error:
2874 free_dma_desc_resources(priv);
5bacd778 2875dma_desc_error:
74371272 2876 phylink_disconnect_phy(priv->phylink);
f66ffe28 2877 return ret;
47dd7a54
GC		 2878}
2879
2880/**
2881 * stmmac_release - close entry point of the driver
2882 * @dev : device pointer.
2883 * Description:
2884 * This is the stop entry point of the driver.
2885 */
2886static int stmmac_release(struct net_device *dev)
2887{
2888 struct stmmac_priv *priv = netdev_priv(dev);
8fce3331 2889 u32 chan;
47dd7a54 2890
d765955d
GC		 2891 if (priv->eee_enabled)
2892 del_timer_sync(&priv->eee_ctrl_timer);
2893
47dd7a54 2894 /* Stop and disconnect the PHY */
74371272
JA		 2895 phylink_stop(priv->phylink);
2896 phylink_disconnect_phy(priv->phylink);
47dd7a54 2897
c22a3f48 2898 stmmac_stop_all_queues(priv);
47dd7a54 2899
c22a3f48 2900 stmmac_disable_all_queues(priv);
47dd7a54 2901
8fce3331
JA		 2902 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
2903 del_timer_sync(&priv->tx_queue[chan].txtimer);
9125cdd1 2904
47dd7a54
GC		 2905 /* Free the IRQ lines */
2906 free_irq(dev->irq, dev);
7a13f8f5
FV		 2907 if (priv->wol_irq != dev->irq)
2908 free_irq(priv->wol_irq, dev);
d7ec8584 2909 if (priv->lpi_irq > 0)
d765955d 2910 free_irq(priv->lpi_irq, dev);
47dd7a54
GC		 2911
2912 /* Stop TX/RX DMA and clear the descriptors */
ae4f0d46 2913 stmmac_stop_all_dma(priv);
47dd7a54
GC		 2914
2915 /* Release and free the Rx/Tx resources */
2916 free_dma_desc_resources(priv);
2917
19449bfc 2918 /* Disable the MAC Rx/Tx */
c10d4c82 2919 stmmac_mac_set(priv, priv->ioaddr, false);
47dd7a54
GC		 2920
2921 netif_carrier_off(dev);
2922
92ba6888
RK		 2923 stmmac_release_ptp(priv);
2924
47dd7a54
GC		 2925 return 0;
2926}
2927
30d93227
JA		 2928static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb,
2929 struct stmmac_tx_queue *tx_q)
2930{
2931 u16 tag = 0x0, inner_tag = 0x0;
2932 u32 inner_type = 0x0;
2933 struct dma_desc *p;
2934
2935 if (!priv->dma_cap.vlins)
2936 return false;
2937 if (!skb_vlan_tag_present(skb))
2938 return false;
2939 if (skb->vlan_proto == htons(ETH_P_8021AD)) {
2940 inner_tag = skb_vlan_tag_get(skb);
2941 inner_type = STMMAC_VLAN_INSERT;
2942 }
2943
2944 tag = skb_vlan_tag_get(skb);
2945
579a25a8
JA		 2946 if (tx_q->tbs & STMMAC_TBS_AVAIL)
2947 p = &tx_q->dma_entx[tx_q->cur_tx].basic;
2948 else
2949 p = &tx_q->dma_tx[tx_q->cur_tx];
2950
30d93227
JA		 2951 if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type))
2952 return false;
2953
2954 stmmac_set_tx_owner(priv, p);
2955 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2956 return true;
2957}
2958
f748be53
AT		 2959/**
2960 * stmmac_tso_allocator - close entry point of the driver
2961 * @priv: driver private structure
2962 * @des: buffer start address
2963 * @total_len: total length to fill in descriptors
2964 * @last_segmant: condition for the last descriptor
ce736788 2965 * @queue: TX queue index
f748be53
AT		 2966 * Description:
2967 * This function fills descriptor and request new descriptors according to
2968 * buffer length to fill
2969 */
a993db88 2970static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des,
ce736788 2971 int total_len, bool last_segment, u32 queue)
f748be53 2972{
ce736788 2973 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
f748be53 2974 struct dma_desc *desc;
5bacd778 2975 u32 buff_size;
ce736788 2976 int tmp_len;
f748be53
AT		 2977
2978 tmp_len = total_len;
2979
2980 while (tmp_len > 0) {
a993db88
JA		 2981 dma_addr_t curr_addr;
2982
ce736788 2983 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
b4c9784c 2984 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
579a25a8
JA		 2985
2986 if (tx_q->tbs & STMMAC_TBS_AVAIL)
2987 desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
2988 else
2989 desc = &tx_q->dma_tx[tx_q->cur_tx];
f748be53 2990
a993db88
JA		 2991 curr_addr = des + (total_len - tmp_len);
2992 if (priv->dma_cap.addr64 <= 32)
2993 desc->des0 = cpu_to_le32(curr_addr);
2994 else
2995 stmmac_set_desc_addr(priv, desc, curr_addr);
2996
f748be53
AT		 2997 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
2998 TSO_MAX_BUFF_SIZE : tmp_len;
2999
42de047d
JA		 3000 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size,
3001 0, 1,
3002 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
3003 0, 0);
f748be53
AT		 3004
3005 tmp_len -= TSO_MAX_BUFF_SIZE;
3006 }
3007}
3008
3009/**
3010 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
3011 * @skb : the socket buffer
3012 * @dev : device pointer
3013 * Description: this is the transmit function that is called on TSO frames
3014 * (support available on GMAC4 and newer chips).
3015 * Diagram below show the ring programming in case of TSO frames:
3016 *
3017 * First Descriptor
3018 * --------
3019 * | DES0 |---> buffer1 = L2/L3/L4 header
3020 * | DES1 |---> TCP Payload (can continue on next descr...)
3021 * | DES2 |---> buffer 1 and 2 len
3022 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
3023 * --------
3024 * |
3025 * ...
3026 * |
3027 * --------
3028 * | DES0 | --| Split TCP Payload on Buffers 1 and 2
3029 * | DES1 | --|
3030 * | DES2 | --> buffer 1 and 2 len
3031 * | DES3 |
3032 * --------
3033 *
3034 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
3035 */
3036static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
3037{
ce736788 3038 struct dma_desc *desc, *first, *mss_desc = NULL;
f748be53 3039 struct stmmac_priv *priv = netdev_priv(dev);
579a25a8 3040 int desc_size, tmp_pay_len = 0, first_tx;
f748be53 3041 int nfrags = skb_shinfo(skb)->nr_frags;
ce736788 3042 u32 queue = skb_get_queue_mapping(skb);
c2837423 3043 unsigned int first_entry, tx_packets;
ce736788 3044 struct stmmac_tx_queue *tx_q;
c2837423 3045 bool has_vlan, set_ic;
579a25a8 3046 u8 proto_hdr_len, hdr;
ce736788 3047 u32 pay_len, mss;
a993db88 3048 dma_addr_t des;
f748be53
AT		 3049 int i;
3050
ce736788 3051 tx_q = &priv->tx_queue[queue];
c2837423 3052 first_tx = tx_q->cur_tx;
ce736788 3053
f748be53 3054 /* Compute header lengths */
b7766206
JA		 3055 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
3056 proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr);
3057 hdr = sizeof(struct udphdr);
3058 } else {
3059 proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
3060 hdr = tcp_hdrlen(skb);
3061 }
f748be53
AT		 3062
3063 /* Desc availability based on threshold should be enough safe */
ce736788 3064 if (unlikely(stmmac_tx_avail(priv, queue) <
f748be53 3065 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
c22a3f48
JP		 3066 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
3067 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
3068 queue));
f748be53 3069 /* This is a hard error, log it. */
38ddc59d
LC		 3070 netdev_err(priv->dev,
3071 "%s: Tx Ring full when queue awake\n",
3072 __func__);
f748be53 3073 }
f748be53
AT		 3074 return NETDEV_TX_BUSY;
3075 }
3076
3077 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
3078
3079 mss = skb_shinfo(skb)->gso_size;
3080
3081 /* set new MSS value if needed */
8d212a9e 3082 if (mss != tx_q->mss) {
579a25a8
JA		 3083 if (tx_q->tbs & STMMAC_TBS_AVAIL)
3084 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
3085 else
3086 mss_desc = &tx_q->dma_tx[tx_q->cur_tx];
3087
42de047d 3088 stmmac_set_mss(priv, mss_desc, mss);
8d212a9e 3089 tx_q->mss = mss;
ce736788 3090 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
b4c9784c 3091 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
f748be53
AT		 3092 }
3093
3094 if (netif_msg_tx_queued(priv)) {
b7766206
JA		 3095 pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
3096 __func__, hdr, proto_hdr_len, pay_len, mss);
f748be53
AT		 3097 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
3098 skb->data_len);
3099 }
3100
30d93227
JA		 3101 /* Check if VLAN can be inserted by HW */
3102 has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
3103
ce736788 3104 first_entry = tx_q->cur_tx;
b4c9784c 3105 WARN_ON(tx_q->tx_skbuff[first_entry]);
f748be53 3106
579a25a8
JA		 3107 if (tx_q->tbs & STMMAC_TBS_AVAIL)
3108 desc = &tx_q->dma_entx[first_entry].basic;
3109 else
3110 desc = &tx_q->dma_tx[first_entry];
f748be53
AT		 3111 first = desc;
3112
30d93227
JA		 3113 if (has_vlan)
3114 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
3115
f748be53
AT		 3116 /* first descriptor: fill Headers on Buf1 */
3117 des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
3118 DMA_TO_DEVICE);
3119 if (dma_mapping_error(priv->device, des))
3120 goto dma_map_err;
3121
ce736788
JP		 3122 tx_q->tx_skbuff_dma[first_entry].buf = des;
3123 tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
f748be53 3124
a993db88
JA		 3125 if (priv->dma_cap.addr64 <= 32) {
3126 first->des0 = cpu_to_le32(des);
f748be53 3127
a993db88
JA		 3128 /* Fill start of payload in buff2 of first descriptor */
3129 if (pay_len)
3130 first->des1 = cpu_to_le32(des + proto_hdr_len);
f748be53 3131
a993db88
JA		 3132 /* If needed take extra descriptors to fill the remaining payload */
3133 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;
3134 } else {
3135 stmmac_set_desc_addr(priv, first, des);
3136 tmp_pay_len = pay_len;
34c15202 3137 des += proto_hdr_len;
b2f07199 3138 pay_len = 0;
a993db88 3139 }
f748be53 3140
ce736788 3141 stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);
f748be53
AT		 3142
3143 /* Prepare fragments */
3144 for (i = 0; i < nfrags; i++) {
3145 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3146
3147 des = skb_frag_dma_map(priv->device, frag, 0,
3148 skb_frag_size(frag),
3149 DMA_TO_DEVICE);
937071c1
TR		 3150 if (dma_mapping_error(priv->device, des))
3151 goto dma_map_err;
f748be53
AT		 3152
3153 stmmac_tso_allocator(priv, des, skb_frag_size(frag),
ce736788 3154 (i == nfrags - 1), queue);
f748be53 3155
ce736788
JP		 3156 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
3157 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
ce736788 3158 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
f748be53
AT		 3159 }
3160
ce736788 3161 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
f748be53 3162
05cf0d1b
NC		 3163 /* Only the last descriptor gets to point to the skb. */
3164 tx_q->tx_skbuff[tx_q->cur_tx] = skb;
3165
7df4a3a7 3166 /* Manage tx mitigation */
c2837423
JA		 3167 tx_packets = (tx_q->cur_tx + 1) - first_tx;
3168 tx_q->tx_count_frames += tx_packets;
3169
3170 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
3171 set_ic = true;
3172 else if (!priv->tx_coal_frames)
3173 set_ic = false;
3174 else if (tx_packets > priv->tx_coal_frames)
3175 set_ic = true;
3176 else if ((tx_q->tx_count_frames % priv->tx_coal_frames) < tx_packets)
3177 set_ic = true;
3178 else
3179 set_ic = false;
3180
3181 if (set_ic) {
579a25a8
JA		 3182 if (tx_q->tbs & STMMAC_TBS_AVAIL)
3183 desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
3184 else
3185 desc = &tx_q->dma_tx[tx_q->cur_tx];
3186
7df4a3a7
JA		 3187 tx_q->tx_count_frames = 0;
3188 stmmac_set_tx_ic(priv, desc);
3189 priv->xstats.tx_set_ic_bit++;
3190 }
3191
05cf0d1b
NC		 3192 /* We've used all descriptors we need for this skb, however,
3193 * advance cur_tx so that it references a fresh descriptor.
3194 * ndo_start_xmit will fill this descriptor the next time it's
3195 * called and stmmac_tx_clean may clean up to this descriptor.
3196 */
ce736788 3197 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
f748be53 3198
ce736788 3199 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
b3e51069
LC		 3200 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
3201 __func__);
c22a3f48 3202 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
f748be53
AT		 3203 }
3204
3205 dev->stats.tx_bytes += skb->len;
3206 priv->xstats.tx_tso_frames++;
3207 priv->xstats.tx_tso_nfrags += nfrags;
3208
8000ddc0
JA		 3209 if (priv->sarc_type)
3210 stmmac_set_desc_sarc(priv, first, priv->sarc_type);
3211
74abc9b1 3212 skb_tx_timestamp(skb);
f748be53
AT		 3213
3214 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3215 priv->hwts_tx_en)) {
3216 /* declare that device is doing timestamping */
3217 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
42de047d 3218 stmmac_enable_tx_timestamp(priv, first);
f748be53
AT		 3219 }
3220
3221 /* Complete the first descriptor before granting the DMA */
42de047d 3222 stmmac_prepare_tso_tx_desc(priv, first, 1,
f748be53
AT		 3223 proto_hdr_len,
3224 pay_len,
ce736788 3225 1, tx_q->tx_skbuff_dma[first_entry].last_segment,
b7766206 3226 hdr / 4, (skb->len - proto_hdr_len));
f748be53
AT		 3227
3228 /* If context desc is used to change MSS */
15d2ee42
NC		 3229 if (mss_desc) {
3230 /* Make sure that first descriptor has been completely
3231 * written, including its own bit. This is because MSS is
3232 * actually before first descriptor, so we need to make
3233 * sure that MSS's own bit is the last thing written.
3234 */
3235 dma_wmb();
42de047d 3236 stmmac_set_tx_owner(priv, mss_desc);
15d2ee42 3237 }
f748be53
AT		 3238
3239 /* The own bit must be the latest setting done when prepare the
3240 * descriptor and then barrier is needed to make sure that
3241 * all is coherent before granting the DMA engine.
3242 */
95eb930a 3243 wmb();
f748be53
AT		 3244
3245 if (netif_msg_pktdata(priv)) {
3246 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
ce736788
JP		 3247 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
3248 tx_q->cur_tx, first, nfrags);
f748be53
AT		 3249 pr_info(">>> frame to be transmitted: ");
3250 print_pkt(skb->data, skb_headlen(skb));
3251 }
3252
c22a3f48 3253 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
f748be53 3254
579a25a8
JA		 3255 if (tx_q->tbs & STMMAC_TBS_AVAIL)
3256 desc_size = sizeof(struct dma_edesc);
3257 else
3258 desc_size = sizeof(struct dma_desc);
3259
3260 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size);
a4e887fa 3261 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
4772f26d 3262 stmmac_tx_timer_arm(priv, queue);
f748be53 3263
f748be53
AT		 3264 return NETDEV_TX_OK;
3265
3266dma_map_err:
f748be53
AT		 3267 dev_err(priv->device, "Tx dma map failed\n");
3268 dev_kfree_skb(skb);
3269 priv->dev->stats.tx_dropped++;
3270 return NETDEV_TX_OK;
3271}
3272
47dd7a54 3273/**
732fdf0e 3274 * stmmac_xmit - Tx entry point of the driver
47dd7a54
GC		 3275 * @skb : the socket buffer
3276 * @dev : device pointer
32ceabca
GC		 3277 * Description : this is the tx entry point of the driver.
3278 * It programs the chain or the ring and supports oversized frames
3279 * and SG feature.
47dd7a54
GC		 3280 */
3281static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
3282{
c2837423 3283 unsigned int first_entry, tx_packets, enh_desc;
47dd7a54 3284 struct stmmac_priv *priv = netdev_priv(dev);
0e80bdc9 3285 unsigned int nopaged_len = skb_headlen(skb);
4a7d666a 3286 int i, csum_insertion = 0, is_jumbo = 0;
ce736788 3287 u32 queue = skb_get_queue_mapping(skb);
47dd7a54 3288 int nfrags = skb_shinfo(skb)->nr_frags;
b7766206 3289 int gso = skb_shinfo(skb)->gso_type;
579a25a8
JA		 3290 struct dma_edesc *tbs_desc = NULL;
3291 int entry, desc_size, first_tx;
47dd7a54 3292 struct dma_desc *desc, *first;
ce736788 3293 struct stmmac_tx_queue *tx_q;
c2837423 3294 bool has_vlan, set_ic;
a993db88 3295 dma_addr_t des;
f748be53 3296
ce736788 3297 tx_q = &priv->tx_queue[queue];
c2837423 3298 first_tx = tx_q->cur_tx;
ce736788 3299
e2cd682d
JA		 3300 if (priv->tx_path_in_lpi_mode)
3301 stmmac_disable_eee_mode(priv);
3302
f748be53
AT		 3303 /* Manage oversized TCP frames for GMAC4 device */
3304 if (skb_is_gso(skb) && priv->tso) {
b7766206
JA		 3305 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))
3306 return stmmac_tso_xmit(skb, dev);
3307 if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4))
f748be53
AT		 3308 return stmmac_tso_xmit(skb, dev);
3309 }
47dd7a54 3310
ce736788 3311 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
c22a3f48
JP		 3312 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
3313 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
3314 queue));
47dd7a54 3315 /* This is a hard error, log it. */
38ddc59d
LC		 3316 netdev_err(priv->dev,
3317 "%s: Tx Ring full when queue awake\n",
3318 __func__);
47dd7a54
GC		 3319 }
3320 return NETDEV_TX_BUSY;
3321 }
3322
30d93227
JA		 3323 /* Check if VLAN can be inserted by HW */
3324 has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
3325
ce736788 3326 entry = tx_q->cur_tx;
0e80bdc9 3327 first_entry = entry;
b4c9784c 3328 WARN_ON(tx_q->tx_skbuff[first_entry]);
47dd7a54 3329
5e982f3b 3330 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
47dd7a54 3331
0e80bdc9 3332 if (likely(priv->extend_desc))
ce736788 3333 desc = (struct dma_desc *)(tx_q->dma_etx + entry);
579a25a8
JA		 3334 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
3335 desc = &tx_q->dma_entx[entry].basic;
c24602ef 3336 else
ce736788 3337 desc = tx_q->dma_tx + entry;
c24602ef 3338
47dd7a54
GC		 3339 first = desc;
3340
30d93227
JA		 3341 if (has_vlan)
3342 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
3343
0e80bdc9 3344 enh_desc = priv->plat->enh_desc;
4a7d666a 3345 /* To program the descriptors according to the size of the frame */
29896a67 3346 if (enh_desc)
2c520b1c 3347 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc);
29896a67 3348
63a550fc 3349 if (unlikely(is_jumbo)) {
2c520b1c 3350 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion);
63a550fc 3351 if (unlikely(entry < 0) && (entry != -EINVAL))
362b37be 3352 goto dma_map_err;
29896a67 3353 }
47dd7a54
GC		 3354
3355 for (i = 0; i < nfrags; i++) {
9e903e08
ED		 3356 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3357 int len = skb_frag_size(frag);
be434d50 3358 bool last_segment = (i == (nfrags - 1));
47dd7a54 3359
e3ad57c9 3360 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
b4c9784c 3361 WARN_ON(tx_q->tx_skbuff[entry]);
e3ad57c9 3362
0e80bdc9 3363 if (likely(priv->extend_desc))
ce736788 3364 desc = (struct dma_desc *)(tx_q->dma_etx + entry);
579a25a8
JA		 3365 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
3366 desc = &tx_q->dma_entx[entry].basic;
c24602ef 3367 else
ce736788 3368 desc = tx_q->dma_tx + entry;
47dd7a54 3369
f748be53
AT		 3370 des = skb_frag_dma_map(priv->device, frag, 0, len,
3371 DMA_TO_DEVICE);
3372 if (dma_mapping_error(priv->device, des))
362b37be
GC		 3373 goto dma_map_err; /* should reuse desc w/o issues */
3374
ce736788 3375 tx_q->tx_skbuff_dma[entry].buf = des;
6844171d
JA		 3376
3377 stmmac_set_desc_addr(priv, desc, des);
f748be53 3378
ce736788
JP		 3379 tx_q->tx_skbuff_dma[entry].map_as_page = true;
3380 tx_q->tx_skbuff_dma[entry].len = len;
3381 tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
0e80bdc9
GC		 3382
3383 /* Prepare the descriptor and set the own bit too */
42de047d
JA		 3384 stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion,
3385 priv->mode, 1, last_segment, skb->len);
47dd7a54
GC		 3386 }
3387
05cf0d1b
NC		 3388 /* Only the last descriptor gets to point to the skb. */
3389 tx_q->tx_skbuff[entry] = skb;
e3ad57c9 3390
7df4a3a7
JA		 3391 /* According to the coalesce parameter the IC bit for the latest
3392 * segment is reset and the timer re-started to clean the tx status.
3393 * This approach takes care about the fragments: desc is the first
3394 * element in case of no SG.
3395 */
c2837423
JA		 3396 tx_packets = (entry + 1) - first_tx;
3397 tx_q->tx_count_frames += tx_packets;
3398
3399 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
3400 set_ic = true;
3401 else if (!priv->tx_coal_frames)
3402 set_ic = false;
3403 else if (tx_packets > priv->tx_coal_frames)
3404 set_ic = true;
3405 else if ((tx_q->tx_count_frames % priv->tx_coal_frames) < tx_packets)
3406 set_ic = true;
3407 else
3408 set_ic = false;
3409
3410 if (set_ic) {
7df4a3a7
JA		 3411 if (likely(priv->extend_desc))
3412 desc = &tx_q->dma_etx[entry].basic;
579a25a8
JA		 3413 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
3414 desc = &tx_q->dma_entx[entry].basic;
7df4a3a7
JA		 3415 else
3416 desc = &tx_q->dma_tx[entry];
3417
3418 tx_q->tx_count_frames = 0;
3419 stmmac_set_tx_ic(priv, desc);
3420 priv->xstats.tx_set_ic_bit++;
3421 }
3422
05cf0d1b
NC		 3423 /* We've used all descriptors we need for this skb, however,
3424 * advance cur_tx so that it references a fresh descriptor.
3425 * ndo_start_xmit will fill this descriptor the next time it's
3426 * called and stmmac_tx_clean may clean up to this descriptor.
3427 */
3428 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
ce736788 3429 tx_q->cur_tx = entry;
47dd7a54 3430
47dd7a54 3431 if (netif_msg_pktdata(priv)) {
38ddc59d
LC		 3432 netdev_dbg(priv->dev,
3433 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
ce736788 3434 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
38ddc59d 3435 entry, first, nfrags);
83d7af64 3436
38ddc59d 3437 netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
47dd7a54
GC		 3438 print_pkt(skb->data, skb->len);
3439 }
0e80bdc9 3440
ce736788 3441 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
b3e51069
LC		 3442 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
3443 __func__);
c22a3f48 3444 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
47dd7a54
GC		 3445 }
3446
3447 dev->stats.tx_bytes += skb->len;
3448
8000ddc0
JA		 3449 if (priv->sarc_type)
3450 stmmac_set_desc_sarc(priv, first, priv->sarc_type);
3451
74abc9b1 3452 skb_tx_timestamp(skb);
3e82ce12 3453
0e80bdc9
GC		 3454 /* Ready to fill the first descriptor and set the OWN bit w/o any
3455 * problems because all the descriptors are actually ready to be
3456 * passed to the DMA engine.
3457 */
3458 if (likely(!is_jumbo)) {
3459 bool last_segment = (nfrags == 0);
3460
f748be53
AT		 3461 des = dma_map_single(priv->device, skb->data,
3462 nopaged_len, DMA_TO_DEVICE);
3463 if (dma_mapping_error(priv->device, des))
0e80bdc9
GC		 3464 goto dma_map_err;
3465
ce736788 3466 tx_q->tx_skbuff_dma[first_entry].buf = des;
6844171d
JA		 3467
3468 stmmac_set_desc_addr(priv, first, des);
f748be53 3469
ce736788
JP		 3470 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
3471 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
0e80bdc9
GC		 3472
3473 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3474 priv->hwts_tx_en)) {
3475 /* declare that device is doing timestamping */
3476 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
42de047d 3477 stmmac_enable_tx_timestamp(priv, first);
0e80bdc9
GC		 3478 }
3479
3480 /* Prepare the first descriptor setting the OWN bit too */
42de047d 3481 stmmac_prepare_tx_desc(priv, first, 1, nopaged_len,
579a25a8 3482 csum_insertion, priv->mode, 0, last_segment,
42de047d 3483 skb->len);
0e80bdc9
GC		 3484 }
3485
579a25a8
JA		 3486 if (tx_q->tbs & STMMAC_TBS_EN) {
3487 struct timespec64 ts = ns_to_timespec64(skb->tstamp);
3488
3489 tbs_desc = &tx_q->dma_entx[first_entry];
3490 stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec);
3491 }
3492
3493 stmmac_set_tx_owner(priv, first);
3494
80acbed9
AK		 3495 /* The own bit must be the latest setting done when prepare the
3496 * descriptor and then barrier is needed to make sure that
3497 * all is coherent before granting the DMA engine.
3498 */
3499 wmb();
3500
c22a3f48 3501 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
f748be53 3502
f1565c60 3503 stmmac_enable_dma_transmission(priv, priv->ioaddr);
8fce3331 3504
579a25a8
JA		 3505 if (likely(priv->extend_desc))
3506 desc_size = sizeof(struct dma_extended_desc);
3507 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
3508 desc_size = sizeof(struct dma_edesc);
3509 else
3510 desc_size = sizeof(struct dma_desc);
3511
3512 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size);
f1565c60 3513 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
4772f26d 3514 stmmac_tx_timer_arm(priv, queue);
52f64fae 3515
362b37be 3516 return NETDEV_TX_OK;
a9097a96 3517
362b37be 3518dma_map_err:
38ddc59d 3519 netdev_err(priv->dev, "Tx DMA map failed\n");
362b37be
GC		 3520 dev_kfree_skb(skb);
3521 priv->dev->stats.tx_dropped++;
47dd7a54
GC		 3522 return NETDEV_TX_OK;
3523}
3524
b9381985
VB		 3525static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
3526{
ab188e8f
EN		 3527 struct vlan_ethhdr *veth;
3528 __be16 vlan_proto;
b9381985
VB		 3529 u16 vlanid;
3530
ab188e8f
EN		 3531 veth = (struct vlan_ethhdr *)skb->data;
3532 vlan_proto = veth->h_vlan_proto;
3533
3534 if ((vlan_proto == htons(ETH_P_8021Q) &&
3535 dev->features & NETIF_F_HW_VLAN_CTAG_RX) ||
3536 (vlan_proto == htons(ETH_P_8021AD) &&
3537 dev->features & NETIF_F_HW_VLAN_STAG_RX)) {
b9381985 3538 /* pop the vlan tag */
ab188e8f
EN		 3539 vlanid = ntohs(veth->h_vlan_TCI);
3540 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2);
b9381985 3541 skb_pull(skb, VLAN_HLEN);
ab188e8f 3542 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid);
b9381985
VB		 3543 }
3544}
3545
3546
54139cf3 3547static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q)
120e87f9 3548{
54139cf3 3549 if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH)
120e87f9
GC		 3550 return 0;
3551
3552 return 1;
3553}
3554
32ceabca 3555/**
732fdf0e 3556 * stmmac_rx_refill - refill used skb preallocated buffers
32ceabca 3557 * @priv: driver private structure
54139cf3 3558 * @queue: RX queue index
32ceabca
GC		 3559 * Description : this is to reallocate the skb for the reception process
3560 * that is based on zero-copy.
3561 */
54139cf3 3562static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
47dd7a54 3563{
54139cf3 3564 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3caa61c2 3565 int len, dirty = stmmac_rx_dirty(priv, queue);
54139cf3
JP		 3566 unsigned int entry = rx_q->dirty_rx;
3567
3caa61c2
JA		 3568 len = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE) * PAGE_SIZE;
3569
e3ad57c9 3570 while (dirty-- > 0) {
2af6106a 3571 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry];
c24602ef 3572 struct dma_desc *p;
d429b66e 3573 bool use_rx_wd;
c24602ef
GC		 3574
3575 if (priv->extend_desc)
54139cf3 3576 p = (struct dma_desc *)(rx_q->dma_erx + entry);
c24602ef 3577 else
54139cf3 3578 p = rx_q->dma_rx + entry;
c24602ef 3579
2af6106a
JA		 3580 if (!buf->page) {
3581 buf->page = page_pool_dev_alloc_pages(rx_q->page_pool);
3582 if (!buf->page)
362b37be 3583 break;
47dd7a54 3584 }
2af6106a 3585
67afd6d1
JA		 3586 if (priv->sph && !buf->sec_page) {
3587 buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool);
3588 if (!buf->sec_page)
3589 break;
3590
3591 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
3592
3593 dma_sync_single_for_device(priv->device, buf->sec_addr,
3594 len, DMA_FROM_DEVICE);
3595 }
3596
2af6106a 3597 buf->addr = page_pool_get_dma_addr(buf->page);
3caa61c2
JA		 3598
3599 /* Sync whole allocation to device. This will invalidate old
3600 * data.
3601 */
3602 dma_sync_single_for_device(priv->device, buf->addr, len,
3603 DMA_FROM_DEVICE);
3604
2af6106a 3605 stmmac_set_desc_addr(priv, p, buf->addr);
67afd6d1 3606 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr);
2af6106a 3607 stmmac_refill_desc3(priv, rx_q, p);
f748be53 3608
d429b66e 3609 rx_q->rx_count_frames++;
6fa9d691
JA		 3610 rx_q->rx_count_frames += priv->rx_coal_frames;
3611 if (rx_q->rx_count_frames > priv->rx_coal_frames)
3612 rx_q->rx_count_frames = 0;
09146abe
JA		 3613
3614 use_rx_wd = !priv->rx_coal_frames;
3615 use_rx_wd |= rx_q->rx_count_frames > 0;
3616 if (!priv->use_riwt)
3617 use_rx_wd = false;
d429b66e 3618
ad688cdb 3619 dma_wmb();
2af6106a 3620 stmmac_set_rx_owner(priv, p, use_rx_wd);
e3ad57c9
GC		 3621
3622 entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE);
47dd7a54 3623 }
54139cf3 3624 rx_q->dirty_rx = entry;
858a31ff
JA		 3625 rx_q->rx_tail_addr = rx_q->dma_rx_phy +
3626 (rx_q->dirty_rx * sizeof(struct dma_desc));
4523a561 3627 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);
47dd7a54
GC		 3628}
3629
88ebe2cf
JA		 3630static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv,
3631 struct dma_desc *p,
3632 int status, unsigned int len)
3633{
3634 int ret, coe = priv->hw->rx_csum;
3635 unsigned int plen = 0, hlen = 0;
3636
3637 /* Not first descriptor, buffer is always zero */
3638 if (priv->sph && len)
3639 return 0;
3640
3641 /* First descriptor, get split header length */
3642 ret = stmmac_get_rx_header_len(priv, p, &hlen);
3643 if (priv->sph && hlen) {
3644 priv->xstats.rx_split_hdr_pkt_n++;
3645 return hlen;
3646 }
3647
3648 /* First descriptor, not last descriptor and not split header */
3649 if (status & rx_not_ls)
3650 return priv->dma_buf_sz;
3651
3652 plen = stmmac_get_rx_frame_len(priv, p, coe);
3653
3654 /* First descriptor and last descriptor and not split header */
3655 return min_t(unsigned int, priv->dma_buf_sz, plen);
3656}
3657
3658static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv,
3659 struct dma_desc *p,
3660 int status, unsigned int len)
3661{
3662 int coe = priv->hw->rx_csum;
3663 unsigned int plen = 0;
3664
3665 /* Not split header, buffer is not available */
3666 if (!priv->sph)
3667 return 0;
3668
3669 /* Not last descriptor */
3670 if (status & rx_not_ls)
3671 return priv->dma_buf_sz;
3672
3673 plen = stmmac_get_rx_frame_len(priv, p, coe);
3674
3675 /* Last descriptor */
3676 return plen - len;
3677}
3678
32ceabca 3679/**
732fdf0e 3680 * stmmac_rx - manage the receive process
32ceabca 3681 * @priv: driver private structure
54139cf3
JP		 3682 * @limit: napi bugget
3683 * @queue: RX queue index.
32ceabca
GC		 3684 * Description : this the function called by the napi poll method.
3685 * It gets all the frames inside the ring.
3686 */
54139cf3 3687static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
47dd7a54 3688{
54139cf3 3689 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
8fce3331 3690 struct stmmac_channel *ch = &priv->channel[queue];
ec222003
JA		 3691 unsigned int count = 0, error = 0, len = 0;
3692 int status = 0, coe = priv->hw->rx_csum;
07b39753 3693 unsigned int next_entry = rx_q->cur_rx;
ec222003 3694 struct sk_buff *skb = NULL;
47dd7a54 3695
83d7af64 3696 if (netif_msg_rx_status(priv)) {
d0225e7d
AT		 3697 void *rx_head;
3698
38ddc59d 3699 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
c24602ef 3700 if (priv->extend_desc)
54139cf3 3701 rx_head = (void *)rx_q->dma_erx;
c24602ef 3702 else
54139cf3 3703 rx_head = (void *)rx_q->dma_rx;
d0225e7d 3704
42de047d 3705 stmmac_display_ring(priv, rx_head, DMA_RX_SIZE, true);
47dd7a54 3706 }
c24602ef 3707 while (count < limit) {
88ebe2cf 3708 unsigned int buf1_len = 0, buf2_len = 0;
ec222003 3709 enum pkt_hash_types hash_type;
2af6106a
JA		 3710 struct stmmac_rx_buffer *buf;
3711 struct dma_desc *np, *p;
ec222003
JA		 3712 int entry;
3713 u32 hash;
47dd7a54 3714
ec222003
JA		 3715 if (!count && rx_q->state_saved) {
3716 skb = rx_q->state.skb;
3717 error = rx_q->state.error;
3718 len = rx_q->state.len;
3719 } else {
3720 rx_q->state_saved = false;
3721 skb = NULL;
3722 error = 0;
3723 len = 0;
3724 }
3725
3726 if (count >= limit)
3727 break;
3728
3729read_again:
88ebe2cf
JA		 3730 buf1_len = 0;
3731 buf2_len = 0;
07b39753 3732 entry = next_entry;
2af6106a 3733 buf = &rx_q->buf_pool[entry];
07b39753 3734
c24602ef 3735 if (priv->extend_desc)
54139cf3 3736 p = (struct dma_desc *)(rx_q->dma_erx + entry);
c24602ef 3737 else
54139cf3 3738 p = rx_q->dma_rx + entry;
c24602ef 3739
c1fa3212 3740 /* read the status of the incoming frame */
42de047d
JA		 3741 status = stmmac_rx_status(priv, &priv->dev->stats,
3742 &priv->xstats, p);
c1fa3212
FG		 3743 /* check if managed by the DMA otherwise go ahead */
3744 if (unlikely(status & dma_own))
47dd7a54
GC		 3745 break;
3746
54139cf3
JP		 3747 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE);
3748 next_entry = rx_q->cur_rx;
e3ad57c9 3749
c24602ef 3750 if (priv->extend_desc)
54139cf3 3751 np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
c24602ef 3752 else
54139cf3 3753 np = rx_q->dma_rx + next_entry;
ba1ffd74
GC		 3754
3755 prefetch(np);
47dd7a54 3756
42de047d
JA		 3757 if (priv->extend_desc)
3758 stmmac_rx_extended_status(priv, &priv->dev->stats,
3759 &priv->xstats, rx_q->dma_erx + entry);
891434b1 3760 if (unlikely(status == discard_frame)) {
2af6106a 3761 page_pool_recycle_direct(rx_q->page_pool, buf->page);
2af6106a 3762 buf->page = NULL;
ec222003 3763 error = 1;
0b273ca4
JA		 3764 if (!priv->hwts_rx_en)
3765 priv->dev->stats.rx_errors++;
ec222003
JA		 3766 }
3767
3768 if (unlikely(error && (status & rx_not_ls)))
3769 goto read_again;
3770 if (unlikely(error)) {
399e06a5 3771 dev_kfree_skb(skb);
88ebe2cf 3772 skb = NULL;
cda4985a 3773 count++;
ec222003
JA		 3774 continue;
3775 }
3776
3777 /* Buffer is good. Go on. */
3778
88ebe2cf
JA		 3779 prefetch(page_address(buf->page));
3780 if (buf->sec_page)
3781 prefetch(page_address(buf->sec_page));
3782
3783 buf1_len = stmmac_rx_buf1_len(priv, p, status, len);
3784 len += buf1_len;
3785 buf2_len = stmmac_rx_buf2_len(priv, p, status, len);
3786 len += buf2_len;
3787
3788 /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
3789 * Type frames (LLC/LLC-SNAP)
3790 *
3791 * llc_snap is never checked in GMAC >= 4, so this ACS
3792 * feature is always disabled and packets need to be
3793 * stripped manually.
3794 */
93b5dce4
JA		 3795 if (likely(!(status & rx_not_ls)) &&
3796 (likely(priv->synopsys_id >= DWMAC_CORE_4_00) ||
3797 unlikely(status != llc_snap))) {
88ebe2cf
JA		 3798 if (buf2_len)
3799 buf2_len -= ETH_FCS_LEN;
3800 else
3801 buf1_len -= ETH_FCS_LEN;
3802
3803 len -= ETH_FCS_LEN;
ec222003 3804 }
22ad3838 3805
ec222003 3806 if (!skb) {
88ebe2cf 3807 skb = napi_alloc_skb(&ch->rx_napi, buf1_len);
ec222003 3808 if (!skb) {
2af6106a 3809 priv->dev->stats.rx_dropped++;
cda4985a 3810 count++;
88ebe2cf 3811 goto drain_data;
47dd7a54 3812 }
47dd7a54 3813
88ebe2cf
JA		 3814 dma_sync_single_for_cpu(priv->device, buf->addr,
3815 buf1_len, DMA_FROM_DEVICE);
2af6106a 3816 skb_copy_to_linear_data(skb, page_address(buf->page),
88ebe2cf
JA		 3817 buf1_len);
3818 skb_put(skb, buf1_len);
2af6106a 3819
ec222003
JA		 3820 /* Data payload copied into SKB, page ready for recycle */
3821 page_pool_recycle_direct(rx_q->page_pool, buf->page);
3822 buf->page = NULL;
88ebe2cf 3823 } else if (buf1_len) {
ec222003 3824 dma_sync_single_for_cpu(priv->device, buf->addr,
88ebe2cf 3825 buf1_len, DMA_FROM_DEVICE);
ec222003 3826 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
88ebe2cf 3827 buf->page, 0, buf1_len,
ec222003 3828 priv->dma_buf_sz);
b9381985 3829
ec222003
JA		 3830 /* Data payload appended into SKB */
3831 page_pool_release_page(rx_q->page_pool, buf->page);
3832 buf->page = NULL;
3833 }
47dd7a54 3834
88ebe2cf 3835 if (buf2_len) {
67afd6d1 3836 dma_sync_single_for_cpu(priv->device, buf->sec_addr,
88ebe2cf 3837 buf2_len, DMA_FROM_DEVICE);
67afd6d1 3838 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
88ebe2cf 3839 buf->sec_page, 0, buf2_len,
67afd6d1
JA		 3840 priv->dma_buf_sz);
3841
67afd6d1
JA		 3842 /* Data payload appended into SKB */
3843 page_pool_release_page(rx_q->page_pool, buf->sec_page);
3844 buf->sec_page = NULL;
3845 }
3846
88ebe2cf 3847drain_data:
ec222003
JA		 3848 if (likely(status & rx_not_ls))
3849 goto read_again;
88ebe2cf
JA		 3850 if (!skb)
3851 continue;
62a2ab93 3852
ec222003 3853 /* Got entire packet into SKB. Finish it. */
76067459 3854
ec222003
JA		 3855 stmmac_get_rx_hwtstamp(priv, p, np, skb);
3856 stmmac_rx_vlan(priv->dev, skb);
3857 skb->protocol = eth_type_trans(skb, priv->dev);
47dd7a54 3858
ec222003
JA		 3859 if (unlikely(!coe))
3860 skb_checksum_none_assert(skb);
3861 else
3862 skb->ip_summed = CHECKSUM_UNNECESSARY;
2af6106a 3863
ec222003
JA		 3864 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))
3865 skb_set_hash(skb, hash, hash_type);
3866
3867 skb_record_rx_queue(skb, queue);
3868 napi_gro_receive(&ch->rx_napi, skb);
88ebe2cf 3869 skb = NULL;
ec222003
JA		 3870
3871 priv->dev->stats.rx_packets++;
3872 priv->dev->stats.rx_bytes += len;
cda4985a 3873 count++;
ec222003
JA		 3874 }
3875
88ebe2cf 3876 if (status & rx_not_ls || skb) {
ec222003
JA		 3877 rx_q->state_saved = true;
3878 rx_q->state.skb = skb;
3879 rx_q->state.error = error;
3880 rx_q->state.len = len;
47dd7a54
GC		 3881 }
3882
54139cf3 3883 stmmac_rx_refill(priv, queue);
47dd7a54
GC		 3884
3885 priv->xstats.rx_pkt_n += count;
3886
3887 return count;
3888}
3889
4ccb4585 3890static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget)
47dd7a54 3891{
8fce3331 3892 struct stmmac_channel *ch =
4ccb4585 3893 container_of(napi, struct stmmac_channel, rx_napi);
8fce3331 3894 struct stmmac_priv *priv = ch->priv_data;
8fce3331 3895 u32 chan = ch->index;
4ccb4585 3896 int work_done;
47dd7a54 3897
9125cdd1 3898 priv->xstats.napi_poll++;
ce736788 3899
4ccb4585 3900 work_done = stmmac_rx(priv, budget, chan);
021bd5e3
JA		 3901 if (work_done < budget && napi_complete_done(napi, work_done)) {
3902 unsigned long flags;
3903
3904 spin_lock_irqsave(&ch->lock, flags);
3905 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0);
3906 spin_unlock_irqrestore(&ch->lock, flags);
3907 }
3908
4ccb4585
JA		 3909 return work_done;
3910}
ce736788 3911
4ccb4585
JA		 3912static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget)
3913{
3914 struct stmmac_channel *ch =
3915 container_of(napi, struct stmmac_channel, tx_napi);
3916 struct stmmac_priv *priv = ch->priv_data;
4ccb4585
JA		 3917 u32 chan = ch->index;
3918 int work_done;
8fce3331 3919
4ccb4585
JA		 3920 priv->xstats.napi_poll++;
3921
3922 work_done = stmmac_tx_clean(priv, DMA_TX_SIZE, chan);
3923 work_done = min(work_done, budget);
8fce3331 3924
021bd5e3
JA		 3925 if (work_done < budget && napi_complete_done(napi, work_done)) {
3926 unsigned long flags;
4ccb4585 3927
021bd5e3
JA		 3928 spin_lock_irqsave(&ch->lock, flags);
3929 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1);
3930 spin_unlock_irqrestore(&ch->lock, flags);
fa0be0a4 3931 }
8fce3331 3932
47dd7a54
GC		 3933 return work_done;
3934}
3935
3936/**
3937 * stmmac_tx_timeout
3938 * @dev : Pointer to net device structure
3939 * Description: this function is called when a packet transmission fails to
7284a3f1 3940 * complete within a reasonable time. The driver will mark the error in the
47dd7a54
GC		 3941 * netdev structure and arrange for the device to be reset to a sane state
3942 * in order to transmit a new packet.
3943 */
0290bd29 3944static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue)
47dd7a54
GC		 3945{
3946 struct stmmac_priv *priv = netdev_priv(dev);
3947
34877a15 3948 stmmac_global_err(priv);
47dd7a54
GC		 3949}
3950
47dd7a54 3951/**
01789349 3952 * stmmac_set_rx_mode - entry point for multicast addressing
47dd7a54
GC		 3953 * @dev : pointer to the device structure
3954 * Description:
3955 * This function is a driver entry point which gets called by the kernel
3956 * whenever multicast addresses must be enabled/disabled.
3957 * Return value:
3958 * void.
3959 */
01789349 3960static void stmmac_set_rx_mode(struct net_device *dev)
47dd7a54
GC		 3961{
3962 struct stmmac_priv *priv = netdev_priv(dev);
3963
c10d4c82 3964 stmmac_set_filter(priv, priv->hw, dev);
47dd7a54
GC		 3965}
3966
3967/**
3968 * stmmac_change_mtu - entry point to change MTU size for the device.
3969 * @dev : device pointer.
3970 * @new_mtu : the new MTU size for the device.
3971 * Description: the Maximum Transfer Unit (MTU) is used by the network layer
3972 * to drive packet transmission. Ethernet has an MTU of 1500 octets
3973 * (ETH_DATA_LEN). This value can be changed with ifconfig.
3974 * Return value:
3975 * 0 on success and an appropriate (-)ve integer as defined in errno.h
3976 * file on failure.
3977 */
3978static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
3979{
38ddc59d 3980 struct stmmac_priv *priv = netdev_priv(dev);
eaf4fac4
JA		 3981 int txfifosz = priv->plat->tx_fifo_size;
3982
3983 if (txfifosz == 0)
3984 txfifosz = priv->dma_cap.tx_fifo_size;
3985
3986 txfifosz /= priv->plat->tx_queues_to_use;
38ddc59d 3987
47dd7a54 3988 if (netif_running(dev)) {
38ddc59d 3989 netdev_err(priv->dev, "must be stopped to change its MTU\n");
47dd7a54
GC		 3990 return -EBUSY;
3991 }
3992
eaf4fac4
JA		 3993 new_mtu = STMMAC_ALIGN(new_mtu);
3994
3995 /* If condition true, FIFO is too small or MTU too large */
3996 if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB))
3997 return -EINVAL;
3998
5e982f3b 3999 dev->mtu = new_mtu;
f748be53 4000
5e982f3b
MM		 4001 netdev_update_features(dev);
4002
4003 return 0;
4004}
4005
c8f44aff 4006static netdev_features_t stmmac_fix_features(struct net_device *dev,
ceb69499 4007 netdev_features_t features)
5e982f3b
MM		 4008{
4009 struct stmmac_priv *priv = netdev_priv(dev);
4010
38912bdb 4011 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
5e982f3b 4012 features &= ~NETIF_F_RXCSUM;
d2afb5bd 4013
5e982f3b 4014 if (!priv->plat->tx_coe)
a188222b 4015 features &= ~NETIF_F_CSUM_MASK;
5e982f3b 4016
ebbb293f
GC		 4017 /* Some GMAC devices have a bugged Jumbo frame support that
4018 * needs to have the Tx COE disabled for oversized frames
4019 * (due to limited buffer sizes). In this case we disable
8d45e42b 4020 * the TX csum insertion in the TDES and not use SF.
ceb69499 4021 */
5e982f3b 4022 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
a188222b 4023 features &= ~NETIF_F_CSUM_MASK;
ebbb293f 4024
f748be53
AT		 4025 /* Disable tso if asked by ethtool */
4026 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
4027 if (features & NETIF_F_TSO)
4028 priv->tso = true;
4029 else
4030 priv->tso = false;
4031 }
4032
5e982f3b 4033 return features;
47dd7a54
GC		 4034}
4035
d2afb5bd
GC		 4036static int stmmac_set_features(struct net_device *netdev,
4037 netdev_features_t features)
4038{
4039 struct stmmac_priv *priv = netdev_priv(netdev);
67afd6d1
JA		 4040 bool sph_en;
4041 u32 chan;
d2afb5bd
GC		 4042
4043 /* Keep the COE Type in case of csum is supporting */
4044 if (features & NETIF_F_RXCSUM)
4045 priv->hw->rx_csum = priv->plat->rx_coe;
4046 else
4047 priv->hw->rx_csum = 0;
4048 /* No check needed because rx_coe has been set before and it will be
4049 * fixed in case of issue.
4050 */
c10d4c82 4051 stmmac_rx_ipc(priv, priv->hw);
d2afb5bd 4052
67afd6d1
JA		 4053 sph_en = (priv->hw->rx_csum > 0) && priv->sph;
4054 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++)
4055 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
4056
d2afb5bd
GC		 4057 return 0;
4058}
4059
32ceabca
GC		 4060/**
4061 * stmmac_interrupt - main ISR
4062 * @irq: interrupt number.
f42234ff 4063 * @dev_id: to pass the net device pointer (must be valid).
32ceabca 4064 * Description: this is the main driver interrupt service routine.
732fdf0e
GC		 4065 * It can call:
4066 * o DMA service routine (to manage incoming frame reception and transmission
4067 * status)
4068 * o Core interrupts to manage: remote wake-up, management counter, LPI
4069 * interrupts.
32ceabca 4070 */
47dd7a54
GC		 4071static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
4072{
4073 struct net_device *dev = (struct net_device *)dev_id;
4074 struct stmmac_priv *priv = netdev_priv(dev);
7bac4e1e
JP		 4075 u32 rx_cnt = priv->plat->rx_queues_to_use;
4076 u32 tx_cnt = priv->plat->tx_queues_to_use;
4077 u32 queues_count;
4078 u32 queue;
7d9e6c5a 4079 bool xmac;
7bac4e1e 4080
7d9e6c5a 4081 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
7bac4e1e 4082 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
47dd7a54 4083
89f7f2cf
SK		 4084 if (priv->irq_wake)
4085 pm_wakeup_event(priv->device, 0);
4086
34877a15
JA		 4087 /* Check if adapter is up */
4088 if (test_bit(STMMAC_DOWN, &priv->state))
4089 return IRQ_HANDLED;
8bf993a5
JA		 4090 /* Check if a fatal error happened */
4091 if (stmmac_safety_feat_interrupt(priv))
4092 return IRQ_HANDLED;
34877a15 4093
d765955d 4094 /* To handle GMAC own interrupts */
7d9e6c5a 4095 if ((priv->plat->has_gmac) || xmac) {
c10d4c82 4096 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats);
61fac60a 4097 int mtl_status;
8f71a88d 4098
d765955d 4099 if (unlikely(status)) {
d765955d 4100 /* For LPI we need to save the tx status */
0982a0f6 4101 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
d765955d 4102 priv->tx_path_in_lpi_mode = true;
0982a0f6 4103 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
d765955d 4104 priv->tx_path_in_lpi_mode = false;
7bac4e1e
JP		 4105 }
4106
61fac60a
JA		 4107 for (queue = 0; queue < queues_count; queue++) {
4108 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
54139cf3 4109
61fac60a
JA		 4110 mtl_status = stmmac_host_mtl_irq_status(priv, priv->hw,
4111 queue);
4112 if (mtl_status != -EINVAL)
4113 status |= mtl_status;
7bac4e1e 4114
61fac60a
JA		 4115 if (status & CORE_IRQ_MTL_RX_OVERFLOW)
4116 stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
4117 rx_q->rx_tail_addr,
4118 queue);
d765955d 4119 }
70523e63
GC		 4120
4121 /* PCS link status */
3fe5cadb 4122 if (priv->hw->pcs) {
70523e63
GC		 4123 if (priv->xstats.pcs_link)
4124 netif_carrier_on(dev);
4125 else
4126 netif_carrier_off(dev);
4127 }
d765955d 4128 }
aec7ff27 4129
d765955d 4130 /* To handle DMA interrupts */
aec7ff27 4131 stmmac_dma_interrupt(priv);
47dd7a54
GC		 4132
4133 return IRQ_HANDLED;
4134}
4135
4136#ifdef CONFIG_NET_POLL_CONTROLLER
4137/* Polling receive - used by NETCONSOLE and other diagnostic tools
ceb69499
GC		 4138 * to allow network I/O with interrupts disabled.
4139 */
47dd7a54
GC		 4140static void stmmac_poll_controller(struct net_device *dev)
4141{
4142 disable_irq(dev->irq);
4143 stmmac_interrupt(dev->irq, dev);
4144 enable_irq(dev->irq);
4145}
4146#endif
4147
4148/**
4149 * stmmac_ioctl - Entry point for the Ioctl
4150 * @dev: Device pointer.
4151 * @rq: An IOCTL specefic structure, that can contain a pointer to
4152 * a proprietary structure used to pass information to the driver.
4153 * @cmd: IOCTL command
4154 * Description:
32ceabca 4155 * Currently it supports the phy_mii_ioctl(...) and HW time stamping.
47dd7a54
GC		 4156 */
4157static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
4158{
74371272 4159 struct stmmac_priv *priv = netdev_priv (dev);
891434b1 4160 int ret = -EOPNOTSUPP;
47dd7a54
GC		 4161
4162 if (!netif_running(dev))
4163 return -EINVAL;
4164
891434b1
RK		 4165 switch (cmd) {
4166 case SIOCGMIIPHY:
4167 case SIOCGMIIREG:
4168 case SIOCSMIIREG:
74371272 4169 ret = phylink_mii_ioctl(priv->phylink, rq, cmd);
891434b1
RK		 4170 break;
4171 case SIOCSHWTSTAMP:
d6228b7c
AP		 4172 ret = stmmac_hwtstamp_set(dev, rq);
4173 break;
4174 case SIOCGHWTSTAMP:
4175 ret = stmmac_hwtstamp_get(dev, rq);
891434b1
RK		 4176 break;
4177 default:
4178 break;
4179 }
28b04113 4180
47dd7a54
GC		 4181 return ret;
4182}
4183
4dbbe8dd
JA		 4184static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
4185 void *cb_priv)
4186{
4187 struct stmmac_priv *priv = cb_priv;
4188 int ret = -EOPNOTSUPP;
4189
425eabdd
JA		 4190 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data))
4191 return ret;
4192
4dbbe8dd
JA		 4193 stmmac_disable_all_queues(priv);
4194
4195 switch (type) {
4196 case TC_SETUP_CLSU32:
425eabdd
JA		 4197 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data);
4198 break;
4199 case TC_SETUP_CLSFLOWER:
4200 ret = stmmac_tc_setup_cls(priv, priv, type_data);
4dbbe8dd
JA		 4201 break;
4202 default:
4203 break;
4204 }
4205
4206 stmmac_enable_all_queues(priv);
4207 return ret;
4208}
4209
955bcb6e
PNA		 4210static LIST_HEAD(stmmac_block_cb_list);
4211
4dbbe8dd
JA		 4212static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type,
4213 void *type_data)
4214{
4215 struct stmmac_priv *priv = netdev_priv(ndev);
4216
4217 switch (type) {
4218 case TC_SETUP_BLOCK:
955bcb6e
PNA		 4219 return flow_block_cb_setup_simple(type_data,
4220 &stmmac_block_cb_list,
4e95bc26
PNA		 4221 stmmac_setup_tc_block_cb,
4222 priv, priv, true);
1f705bc6
JA		 4223 case TC_SETUP_QDISC_CBS:
4224 return stmmac_tc_setup_cbs(priv, priv, type_data);
b60189e0
JA		 4225 case TC_SETUP_QDISC_TAPRIO:
4226 return stmmac_tc_setup_taprio(priv, priv, type_data);
430b383c
JA		 4227 case TC_SETUP_QDISC_ETF:
4228 return stmmac_tc_setup_etf(priv, priv, type_data);
4dbbe8dd
JA		 4229 default:
4230 return -EOPNOTSUPP;
4231 }
4232}
4233
4993e5b3
JA		 4234static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb,
4235 struct net_device *sb_dev)
4236{
b7766206
JA		 4237 int gso = skb_shinfo(skb)->gso_type;
4238
4239 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) {
4993e5b3 4240 /*
b7766206 4241 * There is no way to determine the number of TSO/USO
4993e5b3 4242 * capable Queues. Let's use always the Queue 0
b7766206 4243 * because if TSO/USO is supported then at least this
4993e5b3
JA		 4244 * one will be capable.
4245 */
4246 return 0;
4247 }
4248
4249 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues;
4250}
4251
a830405e
BV		 4252static int stmmac_set_mac_address(struct net_device *ndev, void *addr)
4253{
4254 struct stmmac_priv *priv = netdev_priv(ndev);
4255 int ret = 0;
4256
4257 ret = eth_mac_addr(ndev, addr);
4258 if (ret)
4259 return ret;
4260
c10d4c82 4261 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0);
a830405e
BV		 4262
4263 return ret;
4264}
4265
50fb4f74 4266#ifdef CONFIG_DEBUG_FS
7ac29055 4267static struct dentry *stmmac_fs_dir;
7ac29055 4268
c24602ef 4269static void sysfs_display_ring(void *head, int size, int extend_desc,
ceb69499 4270 struct seq_file *seq)
7ac29055 4271{
7ac29055 4272 int i;
ceb69499
GC		 4273 struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
4274 struct dma_desc *p = (struct dma_desc *)head;
7ac29055 4275
c24602ef 4276 for (i = 0; i < size; i++) {
c24602ef 4277 if (extend_desc) {
c24602ef 4278 seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
ceb69499 4279 i, (unsigned int)virt_to_phys(ep),
f8be0d78
MW		 4280 le32_to_cpu(ep->basic.des0),
4281 le32_to_cpu(ep->basic.des1),
4282 le32_to_cpu(ep->basic.des2),
4283 le32_to_cpu(ep->basic.des3));
c24602ef
GC		 4284 ep++;
4285 } else {
c24602ef 4286 seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
66c25f6e 4287 i, (unsigned int)virt_to_phys(p),
f8be0d78
MW		 4288 le32_to_cpu(p->des0), le32_to_cpu(p->des1),
4289 le32_to_cpu(p->des2), le32_to_cpu(p->des3));
c24602ef
GC		 4290 p++;
4291 }
7ac29055
GC		 4292 seq_printf(seq, "\n");
4293 }
c24602ef 4294}
7ac29055 4295
fb0d9c63 4296static int stmmac_rings_status_show(struct seq_file *seq, void *v)
c24602ef
GC		 4297{
4298 struct net_device *dev = seq->private;
4299 struct stmmac_priv *priv = netdev_priv(dev);
54139cf3 4300 u32 rx_count = priv->plat->rx_queues_to_use;
ce736788 4301 u32 tx_count = priv->plat->tx_queues_to_use;
54139cf3
JP		 4302 u32 queue;
4303
5f2b8b62
TR		 4304 if ((dev->flags & IFF_UP) == 0)
4305 return 0;
4306
54139cf3
JP		 4307 for (queue = 0; queue < rx_count; queue++) {
4308 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4309
4310 seq_printf(seq, "RX Queue %d:\n", queue);
4311
4312 if (priv->extend_desc) {
4313 seq_printf(seq, "Extended descriptor ring:\n");
4314 sysfs_display_ring((void *)rx_q->dma_erx,
4315 DMA_RX_SIZE, 1, seq);
4316 } else {
4317 seq_printf(seq, "Descriptor ring:\n");
4318 sysfs_display_ring((void *)rx_q->dma_rx,
4319 DMA_RX_SIZE, 0, seq);
4320 }
4321 }
aff3d9ef 4322
ce736788
JP		 4323 for (queue = 0; queue < tx_count; queue++) {
4324 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
4325
4326 seq_printf(seq, "TX Queue %d:\n", queue);
4327
4328 if (priv->extend_desc) {
4329 seq_printf(seq, "Extended descriptor ring:\n");
4330 sysfs_display_ring((void *)tx_q->dma_etx,
4331 DMA_TX_SIZE, 1, seq);
579a25a8 4332 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) {
ce736788
JP		 4333 seq_printf(seq, "Descriptor ring:\n");
4334 sysfs_display_ring((void *)tx_q->dma_tx,
4335 DMA_TX_SIZE, 0, seq);
4336 }
7ac29055
GC		 4337 }
4338
4339 return 0;
4340}
fb0d9c63 4341DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status);
7ac29055 4342
fb0d9c63 4343static int stmmac_dma_cap_show(struct seq_file *seq, void *v)
e7434821
GC		 4344{
4345 struct net_device *dev = seq->private;
4346 struct stmmac_priv *priv = netdev_priv(dev);
4347
19e30c14 4348 if (!priv->hw_cap_support) {
e7434821
GC		 4349 seq_printf(seq, "DMA HW features not supported\n");
4350 return 0;
4351 }
4352
4353 seq_printf(seq, "==============================\n");
4354 seq_printf(seq, "\tDMA HW features\n");
4355 seq_printf(seq, "==============================\n");
4356
22d3efe5 4357 seq_printf(seq, "\t10/100 Mbps: %s\n",
e7434821 4358 (priv->dma_cap.mbps_10_100) ? "Y" : "N");
22d3efe5 4359 seq_printf(seq, "\t1000 Mbps: %s\n",
e7434821 4360 (priv->dma_cap.mbps_1000) ? "Y" : "N");
22d3efe5 4361 seq_printf(seq, "\tHalf duplex: %s\n",
e7434821
GC		 4362 (priv->dma_cap.half_duplex) ? "Y" : "N");
4363 seq_printf(seq, "\tHash Filter: %s\n",
4364 (priv->dma_cap.hash_filter) ? "Y" : "N");
4365 seq_printf(seq, "\tMultiple MAC address registers: %s\n",
4366 (priv->dma_cap.multi_addr) ? "Y" : "N");
8d45e42b 4367 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
e7434821
GC		 4368 (priv->dma_cap.pcs) ? "Y" : "N");
4369 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
4370 (priv->dma_cap.sma_mdio) ? "Y" : "N");
4371 seq_printf(seq, "\tPMT Remote wake up: %s\n",
4372 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
4373 seq_printf(seq, "\tPMT Magic Frame: %s\n",
4374 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
4375 seq_printf(seq, "\tRMON module: %s\n",
4376 (priv->dma_cap.rmon) ? "Y" : "N");
4377 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
4378 (priv->dma_cap.time_stamp) ? "Y" : "N");
22d3efe5 4379 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
e7434821 4380 (priv->dma_cap.atime_stamp) ? "Y" : "N");
22d3efe5 4381 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
e7434821
GC		 4382 (priv->dma_cap.eee) ? "Y" : "N");
4383 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
4384 seq_printf(seq, "\tChecksum Offload in TX: %s\n",
4385 (priv->dma_cap.tx_coe) ? "Y" : "N");
f748be53
AT		 4386 if (priv->synopsys_id >= DWMAC_CORE_4_00) {
4387 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
4388 (priv->dma_cap.rx_coe) ? "Y" : "N");
4389 } else {
4390 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
4391 (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
4392 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
4393 (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
4394 }
e7434821
GC		 4395 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
4396 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
4397 seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
4398 priv->dma_cap.number_rx_channel);
4399 seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
4400 priv->dma_cap.number_tx_channel);
7d0b447a
JA		 4401 seq_printf(seq, "\tNumber of Additional RX queues: %d\n",
4402 priv->dma_cap.number_rx_queues);
4403 seq_printf(seq, "\tNumber of Additional TX queues: %d\n",
4404 priv->dma_cap.number_tx_queues);
e7434821
GC		 4405 seq_printf(seq, "\tEnhanced descriptors: %s\n",
4406 (priv->dma_cap.enh_desc) ? "Y" : "N");
7d0b447a
JA		 4407 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size);
4408 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size);
4409 seq_printf(seq, "\tHash Table Size: %d\n", priv->dma_cap.hash_tb_sz);
4410 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N");
4411 seq_printf(seq, "\tNumber of PPS Outputs: %d\n",
4412 priv->dma_cap.pps_out_num);
4413 seq_printf(seq, "\tSafety Features: %s\n",
4414 priv->dma_cap.asp ? "Y" : "N");
4415 seq_printf(seq, "\tFlexible RX Parser: %s\n",
4416 priv->dma_cap.frpsel ? "Y" : "N");
4417 seq_printf(seq, "\tEnhanced Addressing: %d\n",
4418 priv->dma_cap.addr64);
4419 seq_printf(seq, "\tReceive Side Scaling: %s\n",
4420 priv->dma_cap.rssen ? "Y" : "N");
4421 seq_printf(seq, "\tVLAN Hash Filtering: %s\n",
4422 priv->dma_cap.vlhash ? "Y" : "N");
4423 seq_printf(seq, "\tSplit Header: %s\n",
4424 priv->dma_cap.sphen ? "Y" : "N");
4425 seq_printf(seq, "\tVLAN TX Insertion: %s\n",
4426 priv->dma_cap.vlins ? "Y" : "N");
4427 seq_printf(seq, "\tDouble VLAN: %s\n",
4428 priv->dma_cap.dvlan ? "Y" : "N");
4429 seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n",
4430 priv->dma_cap.l3l4fnum);
4431 seq_printf(seq, "\tARP Offloading: %s\n",
4432 priv->dma_cap.arpoffsel ? "Y" : "N");
44e65475
JA		 4433 seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n",
4434 priv->dma_cap.estsel ? "Y" : "N");
4435 seq_printf(seq, "\tFrame Preemption (FPE): %s\n",
4436 priv->dma_cap.fpesel ? "Y" : "N");
4437 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n",
4438 priv->dma_cap.tbssel ? "Y" : "N");
e7434821
GC		 4439 return 0;
4440}
fb0d9c63 4441DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap);
e7434821 4442
481a7d15
JM		 4443/* Use network device events to rename debugfs file entries.
4444 */
4445static int stmmac_device_event(struct notifier_block *unused,
4446 unsigned long event, void *ptr)
4447{
4448 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
4449 struct stmmac_priv *priv = netdev_priv(dev);
4450
4451 if (dev->netdev_ops != &stmmac_netdev_ops)
4452 goto done;
4453
4454 switch (event) {
4455 case NETDEV_CHANGENAME:
4456 if (priv->dbgfs_dir)
4457 priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir,
4458 priv->dbgfs_dir,
4459 stmmac_fs_dir,
4460 dev->name);
4461 break;
4462 }
4463done:
4464 return NOTIFY_DONE;
4465}
4466
4467static struct notifier_block stmmac_notifier = {
4468 .notifier_call = stmmac_device_event,
4469};
4470
8d72ab11 4471static void stmmac_init_fs(struct net_device *dev)
7ac29055 4472{
466c5ac8
MO		 4473 struct stmmac_priv *priv = netdev_priv(dev);
4474
474a31e1
AK		 4475 rtnl_lock();
4476
466c5ac8
MO		 4477 /* Create per netdev entries */
4478 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
7ac29055 4479
7ac29055 4480 /* Entry to report DMA RX/TX rings */
8d72ab11
GKH		 4481 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev,
4482 &stmmac_rings_status_fops);
7ac29055 4483
e7434821 4484 /* Entry to report the DMA HW features */
8d72ab11
GKH		 4485 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev,
4486 &stmmac_dma_cap_fops);
481a7d15 4487
474a31e1 4488 rtnl_unlock();
7ac29055
GC		 4489}
4490
466c5ac8 4491static void stmmac_exit_fs(struct net_device *dev)
7ac29055 4492{
466c5ac8
MO		 4493 struct stmmac_priv *priv = netdev_priv(dev);
4494
4495 debugfs_remove_recursive(priv->dbgfs_dir);
7ac29055 4496}
50fb4f74 4497#endif /* CONFIG_DEBUG_FS */
7ac29055 4498
3cd1cfcb
JA		 4499static u32 stmmac_vid_crc32_le(__le16 vid_le)
4500{
4501 unsigned char *data = (unsigned char *)&vid_le;
4502 unsigned char data_byte = 0;
4503 u32 crc = ~0x0;
4504 u32 temp = 0;
4505 int i, bits;
4506
4507 bits = get_bitmask_order(VLAN_VID_MASK);
4508 for (i = 0; i < bits; i++) {
4509 if ((i % 8) == 0)
4510 data_byte = data[i / 8];
4511
4512 temp = ((crc & 1) ^ data_byte) & 1;
4513 crc >>= 1;
4514 data_byte >>= 1;
4515
4516 if (temp)
4517 crc ^= 0xedb88320;
4518 }
4519
4520 return crc;
4521}
4522
4523static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double)
4524{
4525 u32 crc, hash = 0;
a24cae70 4526 __le16 pmatch = 0;
c7ab0b80
JA		 4527 int count = 0;
4528 u16 vid = 0;
3cd1cfcb
JA		 4529
4530 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
4531 __le16 vid_le = cpu_to_le16(vid);
4532 crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28;
4533 hash |= (1 << crc);
c7ab0b80
JA		 4534 count++;
4535 }
4536
4537 if (!priv->dma_cap.vlhash) {
4538 if (count > 2) /* VID = 0 always passes filter */
4539 return -EOPNOTSUPP;
4540
a24cae70 4541 pmatch = cpu_to_le16(vid);
c7ab0b80 4542 hash = 0;
3cd1cfcb
JA		 4543 }
4544
a24cae70 4545 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double);
3cd1cfcb
JA		 4546}
4547
4548static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid)
4549{
4550 struct stmmac_priv *priv = netdev_priv(ndev);
4551 bool is_double = false;
4552 int ret;
4553
3cd1cfcb
JA		 4554 if (be16_to_cpu(proto) == ETH_P_8021AD)
4555 is_double = true;
4556
4557 set_bit(vid, priv->active_vlans);
4558 ret = stmmac_vlan_update(priv, is_double);
4559 if (ret) {
4560 clear_bit(vid, priv->active_vlans);
4561 return ret;
4562 }
4563
dd6a4998
JA		 4564 if (priv->hw->num_vlan) {
4565 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);
4566 if (ret)
4567 return ret;
4568 }
ed64639b 4569
dd6a4998 4570 return 0;
3cd1cfcb
JA		 4571}
4572
4573static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid)
4574{
4575 struct stmmac_priv *priv = netdev_priv(ndev);
4576 bool is_double = false;
ed64639b 4577 int ret;
3cd1cfcb 4578
3cd1cfcb
JA		 4579 if (be16_to_cpu(proto) == ETH_P_8021AD)
4580 is_double = true;
4581
4582 clear_bit(vid, priv->active_vlans);
dd6a4998
JA		 4583
4584 if (priv->hw->num_vlan) {
4585 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);
4586 if (ret)
4587 return ret;
4588 }
ed64639b 4589
3cd1cfcb
JA		 4590 return stmmac_vlan_update(priv, is_double);
4591}
4592
47dd7a54
GC		 4593static const struct net_device_ops stmmac_netdev_ops = {
4594 .ndo_open = stmmac_open,
4595 .ndo_start_xmit = stmmac_xmit,
4596 .ndo_stop = stmmac_release,
4597 .ndo_change_mtu = stmmac_change_mtu,
5e982f3b 4598 .ndo_fix_features = stmmac_fix_features,
d2afb5bd 4599 .ndo_set_features = stmmac_set_features,
01789349 4600 .ndo_set_rx_mode = stmmac_set_rx_mode,
47dd7a54
GC		 4601 .ndo_tx_timeout = stmmac_tx_timeout,
4602 .ndo_do_ioctl = stmmac_ioctl,
4dbbe8dd 4603 .ndo_setup_tc = stmmac_setup_tc,
4993e5b3 4604 .ndo_select_queue = stmmac_select_queue,
47dd7a54
GC		 4605#ifdef CONFIG_NET_POLL_CONTROLLER
4606 .ndo_poll_controller = stmmac_poll_controller,
4607#endif
a830405e 4608 .ndo_set_mac_address = stmmac_set_mac_address,
3cd1cfcb
JA		 4609 .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid,
4610 .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid,
47dd7a54
GC		 4611};
4612
34877a15
JA		 4613static void stmmac_reset_subtask(struct stmmac_priv *priv)
4614{
4615 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state))
4616 return;
4617 if (test_bit(STMMAC_DOWN, &priv->state))
4618 return;
4619
4620 netdev_err(priv->dev, "Reset adapter.\n");
4621
4622 rtnl_lock();
4623 netif_trans_update(priv->dev);
4624 while (test_and_set_bit(STMMAC_RESETING, &priv->state))
4625 usleep_range(1000, 2000);
4626
4627 set_bit(STMMAC_DOWN, &priv->state);
4628 dev_close(priv->dev);
00f54e68 4629 dev_open(priv->dev, NULL);
34877a15
JA		 4630 clear_bit(STMMAC_DOWN, &priv->state);
4631 clear_bit(STMMAC_RESETING, &priv->state);
4632 rtnl_unlock();
4633}
4634
4635static void stmmac_service_task(struct work_struct *work)
4636{
4637 struct stmmac_priv *priv = container_of(work, struct stmmac_priv,
4638 service_task);
4639
4640 stmmac_reset_subtask(priv);
4641 clear_bit(STMMAC_SERVICE_SCHED, &priv->state);
4642}
4643
cf3f047b
GC		 4644/**
4645 * stmmac_hw_init - Init the MAC device
32ceabca 4646 * @priv: driver private structure
732fdf0e
GC		 4647 * Description: this function is to configure the MAC device according to
4648 * some platform parameters or the HW capability register. It prepares the
4649 * driver to use either ring or chain modes and to setup either enhanced or
4650 * normal descriptors.
cf3f047b
GC		 4651 */
4652static int stmmac_hw_init(struct stmmac_priv *priv)
4653{
5f0456b4 4654 int ret;
cf3f047b 4655
9f93ac8d
LC		 4656 /* dwmac-sun8i only work in chain mode */
4657 if (priv->plat->has_sun8i)
4658 chain_mode = 1;
5f0456b4 4659 priv->chain_mode = chain_mode;
9f93ac8d 4660
5f0456b4
JA		 4661 /* Initialize HW Interface */
4662 ret = stmmac_hwif_init(priv);
4663 if (ret)
4664 return ret;
4a7d666a 4665
cf3f047b
GC		 4666 /* Get the HW capability (new GMAC newer than 3.50a) */
4667 priv->hw_cap_support = stmmac_get_hw_features(priv);
4668 if (priv->hw_cap_support) {
38ddc59d 4669 dev_info(priv->device, "DMA HW capability register supported\n");
cf3f047b
GC		 4670
4671 /* We can override some gmac/dma configuration fields: e.g.
4672 * enh_desc, tx_coe (e.g. that are passed through the
4673 * platform) with the values from the HW capability
4674 * register (if supported).
4675 */
4676 priv->plat->enh_desc = priv->dma_cap.enh_desc;
cf3f047b 4677 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
3fe5cadb 4678 priv->hw->pmt = priv->plat->pmt;
b8ef7020
BH		 4679 if (priv->dma_cap.hash_tb_sz) {
4680 priv->hw->multicast_filter_bins =
4681 (BIT(priv->dma_cap.hash_tb_sz) << 5);
4682 priv->hw->mcast_bits_log2 =
4683 ilog2(priv->hw->multicast_filter_bins);
4684 }
38912bdb 4685
a8df35d4
EG		 4686 /* TXCOE doesn't work in thresh DMA mode */
4687 if (priv->plat->force_thresh_dma_mode)
4688 priv->plat->tx_coe = 0;
4689 else
4690 priv->plat->tx_coe = priv->dma_cap.tx_coe;
4691
f748be53
AT		 4692 /* In case of GMAC4 rx_coe is from HW cap register. */
4693 priv->plat->rx_coe = priv->dma_cap.rx_coe;
38912bdb
DS		 4694
4695 if (priv->dma_cap.rx_coe_type2)
4696 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
4697 else if (priv->dma_cap.rx_coe_type1)
4698 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
4699
38ddc59d
LC		 4700 } else {
4701 dev_info(priv->device, "No HW DMA feature register supported\n");
4702 }
cf3f047b 4703
d2afb5bd
GC		 4704 if (priv->plat->rx_coe) {
4705 priv->hw->rx_csum = priv->plat->rx_coe;
38ddc59d 4706 dev_info(priv->device, "RX Checksum Offload Engine supported\n");
f748be53 4707 if (priv->synopsys_id < DWMAC_CORE_4_00)
38ddc59d 4708 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
d2afb5bd 4709 }
cf3f047b 4710 if (priv->plat->tx_coe)
38ddc59d 4711 dev_info(priv->device, "TX Checksum insertion supported\n");
cf3f047b
GC		 4712
4713 if (priv->plat->pmt) {
38ddc59d 4714 dev_info(priv->device, "Wake-Up On Lan supported\n");
cf3f047b
GC		 4715 device_set_wakeup_capable(priv->device, 1);
4716 }
4717
f748be53 4718 if (priv->dma_cap.tsoen)
38ddc59d 4719 dev_info(priv->device, "TSO supported\n");
f748be53 4720
7cfde0af
JA		 4721 /* Run HW quirks, if any */
4722 if (priv->hwif_quirks) {
4723 ret = priv->hwif_quirks(priv);
4724 if (ret)
4725 return ret;
4726 }
4727
3b509466
JA		 4728 /* Rx Watchdog is available in the COREs newer than the 3.40.
4729 * In some case, for example on bugged HW this feature
4730 * has to be disable and this can be done by passing the
4731 * riwt_off field from the platform.
4732 */
4733 if (((priv->synopsys_id >= DWMAC_CORE_3_50) ||
4734 (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) {
4735 priv->use_riwt = 1;
4736 dev_info(priv->device,
4737 "Enable RX Mitigation via HW Watchdog Timer\n");
4738 }
4739
c24602ef 4740 return 0;
cf3f047b
GC		 4741}
4742
47dd7a54 4743/**
bfab27a1
GC		 4744 * stmmac_dvr_probe
4745 * @device: device pointer
ff3dd78c 4746 * @plat_dat: platform data pointer
e56788cf 4747 * @res: stmmac resource pointer
bfab27a1
GC		 4748 * Description: this is the main probe function used to
4749 * call the alloc_etherdev, allocate the priv structure.
9afec6ef 4750 * Return:
15ffac73 4751 * returns 0 on success, otherwise errno.
47dd7a54 4752 */
15ffac73
JE		 4753int stmmac_dvr_probe(struct device *device,
4754 struct plat_stmmacenet_data *plat_dat,
4755 struct stmmac_resources *res)
47dd7a54 4756{
bfab27a1
GC		 4757 struct net_device *ndev = NULL;
4758 struct stmmac_priv *priv;
76067459
JA		 4759 u32 queue, rxq, maxq;
4760 int i, ret = 0;
47dd7a54 4761
9737070c
JZ		 4762 ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv),
4763 MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES);
41de8d4c 4764 if (!ndev)
15ffac73 4765 return -ENOMEM;
bfab27a1
GC		 4766
4767 SET_NETDEV_DEV(ndev, device);
4768
4769 priv = netdev_priv(ndev);
4770 priv->device = device;
4771 priv->dev = ndev;
47dd7a54 4772
bfab27a1 4773 stmmac_set_ethtool_ops(ndev);
cf3f047b
GC		 4774 priv->pause = pause;
4775 priv->plat = plat_dat;
e56788cf
JE		 4776 priv->ioaddr = res->addr;
4777 priv->dev->base_addr = (unsigned long)res->addr;
4778
4779 priv->dev->irq = res->irq;
4780 priv->wol_irq = res->wol_irq;
4781 priv->lpi_irq = res->lpi_irq;
4782
a51645f7 4783 if (!IS_ERR_OR_NULL(res->mac))
e56788cf 4784 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
cf3f047b 4785
a7a62685 4786 dev_set_drvdata(device, priv->dev);
803f8fc4 4787
cf3f047b
GC		 4788 /* Verify driver arguments */
4789 stmmac_verify_args();
bfab27a1 4790
34877a15
JA		 4791 /* Allocate workqueue */
4792 priv->wq = create_singlethread_workqueue("stmmac_wq");
4793 if (!priv->wq) {
4794 dev_err(priv->device, "failed to create workqueue\n");
9737070c 4795 return -ENOMEM;
34877a15
JA		 4796 }
4797
4798 INIT_WORK(&priv->service_task, stmmac_service_task);
4799
cf3f047b 4800 /* Override with kernel parameters if supplied XXX CRS XXX
ceb69499
GC		 4801 * this needs to have multiple instances
4802 */
cf3f047b
GC		 4803 if ((phyaddr >= 0) && (phyaddr <= 31))
4804 priv->plat->phy_addr = phyaddr;
4805
90f522a2
EP		 4806 if (priv->plat->stmmac_rst) {
4807 ret = reset_control_assert(priv->plat->stmmac_rst);
f573c0b9 4808 reset_control_deassert(priv->plat->stmmac_rst);
90f522a2
EP		 4809 /* Some reset controllers have only reset callback instead of
4810 * assert + deassert callbacks pair.
4811 */
4812 if (ret == -ENOTSUPP)
4813 reset_control_reset(priv->plat->stmmac_rst);
4814 }
c5e4ddbd 4815
cf3f047b 4816 /* Init MAC and get the capabilities */
c24602ef
GC		 4817 ret = stmmac_hw_init(priv);
4818 if (ret)
62866e98 4819 goto error_hw_init;
cf3f047b 4820
b561af36
VK		 4821 stmmac_check_ether_addr(priv);
4822
c22a3f48 4823 /* Configure real RX and TX queues */
c02b7a91
JP		 4824 netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use);
4825 netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use);
c22a3f48 4826
cf3f047b 4827 ndev->netdev_ops = &stmmac_netdev_ops;
bfab27a1 4828
cf3f047b
GC		 4829 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4830 NETIF_F_RXCSUM;
f748be53 4831
4dbbe8dd
JA		 4832 ret = stmmac_tc_init(priv, priv);
4833 if (!ret) {
4834 ndev->hw_features |= NETIF_F_HW_TC;
4835 }
4836
f748be53 4837 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
9edfa7da 4838 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
b7766206
JA		 4839 if (priv->plat->has_gmac4)
4840 ndev->hw_features |= NETIF_F_GSO_UDP_L4;
f748be53 4841 priv->tso = true;
38ddc59d 4842 dev_info(priv->device, "TSO feature enabled\n");
f748be53 4843 }
a993db88 4844
67afd6d1
JA		 4845 if (priv->dma_cap.sphen) {
4846 ndev->hw_features |= NETIF_F_GRO;
4847 priv->sph = true;
4848 dev_info(priv->device, "SPH feature enabled\n");
4849 }
4850
a993db88
JA		 4851 if (priv->dma_cap.addr64) {
4852 ret = dma_set_mask_and_coherent(device,
4853 DMA_BIT_MASK(priv->dma_cap.addr64));
4854 if (!ret) {
4855 dev_info(priv->device, "Using %d bits DMA width\n",
4856 priv->dma_cap.addr64);
968a2978
TR		 4857
4858 /*
4859 * If more than 32 bits can be addressed, make sure to
4860 * enable enhanced addressing mode.
4861 */
4862 if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT))
4863 priv->plat->dma_cfg->eame = true;
a993db88
JA		 4864 } else {
4865 ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32));
4866 if (ret) {
4867 dev_err(priv->device, "Failed to set DMA Mask\n");
4868 goto error_hw_init;
4869 }
4870
4871 priv->dma_cap.addr64 = 32;
4872 }
4873 }
4874
bfab27a1
GC		 4875 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
4876 ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
47dd7a54
GC		 4877#ifdef STMMAC_VLAN_TAG_USED
4878 /* Both mac100 and gmac support receive VLAN tag detection */
ab188e8f 4879 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX;
3cd1cfcb
JA		 4880 if (priv->dma_cap.vlhash) {
4881 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4882 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER;
4883 }
30d93227
JA		 4884 if (priv->dma_cap.vlins) {
4885 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX;
4886 if (priv->dma_cap.dvlan)
4887 ndev->features |= NETIF_F_HW_VLAN_STAG_TX;
4888 }
47dd7a54
GC		 4889#endif
4890 priv->msg_enable = netif_msg_init(debug, default_msg_level);
4891
76067459
JA		 4892 /* Initialize RSS */
4893 rxq = priv->plat->rx_queues_to_use;
4894 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key));
4895 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)
4896 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq);
4897
4898 if (priv->dma_cap.rssen && priv->plat->rss_en)
4899 ndev->features |= NETIF_F_RXHASH;
4900
44770e11
JW		 4901 /* MTU range: 46 - hw-specific max */
4902 ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
56bcd591 4903 if (priv->plat->has_xgmac)
7d9e6c5a 4904 ndev->max_mtu = XGMAC_JUMBO_LEN;
56bcd591
JA		 4905 else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
4906 ndev->max_mtu = JUMBO_LEN;
44770e11
JW		 4907 else
4908 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
a2cd64f3
KHL		 4909 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
4910 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
4911 */
4912 if ((priv->plat->maxmtu < ndev->max_mtu) &&
4913 (priv->plat->maxmtu >= ndev->min_mtu))
44770e11 4914 ndev->max_mtu = priv->plat->maxmtu;
a2cd64f3 4915 else if (priv->plat->maxmtu < ndev->min_mtu)
b618ab45
HK		 4916 dev_warn(priv->device,
4917 "%s: warning: maxmtu having invalid value (%d)\n",
4918 __func__, priv->plat->maxmtu);
44770e11 4919
47dd7a54
GC		 4920 if (flow_ctrl)
4921 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */
4922
8fce3331
JA		 4923 /* Setup channels NAPI */
4924 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
c22a3f48 4925
8fce3331
JA		 4926 for (queue = 0; queue < maxq; queue++) {
4927 struct stmmac_channel *ch = &priv->channel[queue];
4928
021bd5e3 4929 spin_lock_init(&ch->lock);
8fce3331
JA		 4930 ch->priv_data = priv;
4931 ch->index = queue;
4932
4ccb4585
JA		 4933 if (queue < priv->plat->rx_queues_to_use) {
4934 netif_napi_add(ndev, &ch->rx_napi, stmmac_napi_poll_rx,
4935 NAPI_POLL_WEIGHT);
4936 }
4937 if (queue < priv->plat->tx_queues_to_use) {
4d97972b
FI		 4938 netif_tx_napi_add(ndev, &ch->tx_napi,
4939 stmmac_napi_poll_tx,
4940 NAPI_POLL_WEIGHT);
4ccb4585 4941 }
c22a3f48 4942 }
47dd7a54 4943
29555fa3 4944 mutex_init(&priv->lock);
f8e96161 4945
cd7201f4
GC		 4946 /* If a specific clk_csr value is passed from the platform
4947 * this means that the CSR Clock Range selection cannot be
4948 * changed at run-time and it is fixed. Viceversa the driver'll try to
4949 * set the MDC clock dynamically according to the csr actual
4950 * clock input.
4951 */
5e7f7fc5 4952 if (priv->plat->clk_csr >= 0)
cd7201f4 4953 priv->clk_csr = priv->plat->clk_csr;
5e7f7fc5
BH		 4954 else
4955 stmmac_clk_csr_set(priv);
cd7201f4 4956
e58bb43f
GC		 4957 stmmac_check_pcs_mode(priv);
4958
a47b9e15 4959 if (priv->hw->pcs != STMMAC_PCS_TBI &&
3fe5cadb 4960 priv->hw->pcs != STMMAC_PCS_RTBI) {
e58bb43f
GC		 4961 /* MDIO bus Registration */
4962 ret = stmmac_mdio_register(ndev);
4963 if (ret < 0) {
b618ab45
HK		 4964 dev_err(priv->device,
4965 "%s: MDIO bus (id: %d) registration failed",
4966 __func__, priv->plat->bus_id);
e58bb43f
GC		 4967 goto error_mdio_register;
4968 }
4bfcbd7a
FV		 4969 }
4970
74371272
JA		 4971 ret = stmmac_phy_setup(priv);
4972 if (ret) {
4973 netdev_err(ndev, "failed to setup phy (%d)\n", ret);
4974 goto error_phy_setup;
4975 }
4976
57016590 4977 ret = register_netdev(ndev);
b2eb09af 4978 if (ret) {
b618ab45
HK		 4979 dev_err(priv->device, "%s: ERROR %i registering the device\n",
4980 __func__, ret);
b2eb09af
FF		 4981 goto error_netdev_register;
4982 }
57016590 4983
b9663b7c
VW		 4984 if (priv->plat->serdes_powerup) {
4985 ret = priv->plat->serdes_powerup(ndev,
4986 priv->plat->bsp_priv);
4987
4988 if (ret < 0)
ab1c637c 4989 goto error_serdes_powerup;
b9663b7c
VW		 4990 }
4991
5f2b8b62 4992#ifdef CONFIG_DEBUG_FS
8d72ab11 4993 stmmac_init_fs(ndev);
5f2b8b62
TR		 4994#endif
4995
57016590 4996 return ret;
47dd7a54 4997
ab1c637c
AS		 4998error_serdes_powerup:
4999 unregister_netdev(ndev);
6a81c26f 5000error_netdev_register:
74371272
JA		 5001 phylink_destroy(priv->phylink);
5002error_phy_setup:
a47b9e15 5003 if (priv->hw->pcs != STMMAC_PCS_TBI &&
b2eb09af
FF		 5004 priv->hw->pcs != STMMAC_PCS_RTBI)
5005 stmmac_mdio_unregister(ndev);
6a81c26f 5006error_mdio_register:
8fce3331
JA		 5007 for (queue = 0; queue < maxq; queue++) {
5008 struct stmmac_channel *ch = &priv->channel[queue];
c22a3f48 5009
4ccb4585
JA		 5010 if (queue < priv->plat->rx_queues_to_use)
5011 netif_napi_del(&ch->rx_napi);
5012 if (queue < priv->plat->tx_queues_to_use)
5013 netif_napi_del(&ch->tx_napi);
c22a3f48 5014 }
62866e98 5015error_hw_init:
34877a15 5016 destroy_workqueue(priv->wq);
47dd7a54 5017
15ffac73 5018 return ret;
47dd7a54 5019}
b2e2f0c7 5020EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
47dd7a54
GC		 5021
5022/**
5023 * stmmac_dvr_remove
f4e7bd81 5024 * @dev: device pointer
47dd7a54 5025 * Description: this function resets the TX/RX processes, disables the MAC RX/TX
bfab27a1 5026 * changes the link status, releases the DMA descriptor rings.
47dd7a54 5027 */
f4e7bd81 5028int stmmac_dvr_remove(struct device *dev)
47dd7a54 5029{
f4e7bd81 5030 struct net_device *ndev = dev_get_drvdata(dev);
aec7ff27 5031 struct stmmac_priv *priv = netdev_priv(ndev);
47dd7a54 5032
38ddc59d 5033 netdev_info(priv->dev, "%s: removing driver", __func__);
47dd7a54 5034
ae4f0d46 5035 stmmac_stop_all_dma(priv);
47dd7a54 5036
b9663b7c
VW		 5037 if (priv->plat->serdes_powerdown)
5038 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv);
5039
c10d4c82 5040 stmmac_mac_set(priv, priv->ioaddr, false);
47dd7a54 5041 netif_carrier_off(ndev);
47dd7a54 5042 unregister_netdev(ndev);
474a31e1
AK		 5043#ifdef CONFIG_DEBUG_FS
5044 stmmac_exit_fs(ndev);
5045#endif
74371272 5046 phylink_destroy(priv->phylink);
f573c0b9 5047 if (priv->plat->stmmac_rst)
5048 reset_control_assert(priv->plat->stmmac_rst);
5049 clk_disable_unprepare(priv->plat->pclk);
5050 clk_disable_unprepare(priv->plat->stmmac_clk);
a47b9e15 5051 if (priv->hw->pcs != STMMAC_PCS_TBI &&
3fe5cadb 5052 priv->hw->pcs != STMMAC_PCS_RTBI)
e743471f 5053 stmmac_mdio_unregister(ndev);
34877a15 5054 destroy_workqueue(priv->wq);
29555fa3 5055 mutex_destroy(&priv->lock);
47dd7a54
GC		 5056
5057 return 0;
5058}
b2e2f0c7 5059EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
47dd7a54 5060
732fdf0e
GC		 5061/**
5062 * stmmac_suspend - suspend callback
f4e7bd81 5063 * @dev: device pointer
732fdf0e
GC		 5064 * Description: this is the function to suspend the device and it is called
5065 * by the platform driver to stop the network queue, release the resources,
5066 * program the PMT register (for WoL), clean and release driver resources.
5067 */
f4e7bd81 5068int stmmac_suspend(struct device *dev)
47dd7a54 5069{
f4e7bd81 5070 struct net_device *ndev = dev_get_drvdata(dev);
874bd42d 5071 struct stmmac_priv *priv = netdev_priv(ndev);
14b41a29 5072 u32 chan;
47dd7a54 5073
874bd42d 5074 if (!ndev || !netif_running(ndev))
47dd7a54
GC		 5075 return 0;
5076
3e2bf04f 5077 phylink_mac_change(priv->phylink, false);
47dd7a54 5078
134cc4ce 5079 mutex_lock(&priv->lock);
19e13cb2 5080
874bd42d 5081 netif_device_detach(ndev);
c22a3f48 5082 stmmac_stop_all_queues(priv);
47dd7a54 5083
c22a3f48 5084 stmmac_disable_all_queues(priv);
874bd42d 5085
14b41a29
NC		 5086 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
5087 del_timer_sync(&priv->tx_queue[chan].txtimer);
5088
874bd42d 5089 /* Stop TX/RX DMA */
ae4f0d46 5090 stmmac_stop_all_dma(priv);
c24602ef 5091
b9663b7c
VW		 5092 if (priv->plat->serdes_powerdown)
5093 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv);
5094
874bd42d 5095 /* Enable Power down mode by programming the PMT regs */
89f7f2cf 5096 if (device_may_wakeup(priv->device)) {
c10d4c82 5097 stmmac_pmt(priv, priv->hw, priv->wolopts);
89f7f2cf
SK		 5098 priv->irq_wake = 1;
5099 } else {
134cc4ce 5100 mutex_unlock(&priv->lock);
3e2bf04f
JA		 5101 rtnl_lock();
5102 phylink_stop(priv->phylink);
5103 rtnl_unlock();
134cc4ce 5104 mutex_lock(&priv->lock);
3e2bf04f 5105
c10d4c82 5106 stmmac_mac_set(priv, priv->ioaddr, false);
db88f10a 5107 pinctrl_pm_select_sleep_state(priv->device);
ba1377ff 5108 /* Disable clock in case of PWM is off */
e497c20e
BH		 5109 if (priv->plat->clk_ptp_ref)
5110 clk_disable_unprepare(priv->plat->clk_ptp_ref);
5111 clk_disable_unprepare(priv->plat->pclk);
5112 clk_disable_unprepare(priv->plat->stmmac_clk);
ba1377ff 5113 }
29555fa3 5114 mutex_unlock(&priv->lock);
2d871aa0 5115
bd00632c 5116 priv->speed = SPEED_UNKNOWN;
47dd7a54
GC		 5117 return 0;
5118}
b2e2f0c7 5119EXPORT_SYMBOL_GPL(stmmac_suspend);
47dd7a54 5120
54139cf3
JP		 5121/**
5122 * stmmac_reset_queues_param - reset queue parameters
5123 * @dev: device pointer
5124 */
5125static void stmmac_reset_queues_param(struct stmmac_priv *priv)
5126{
5127 u32 rx_cnt = priv->plat->rx_queues_to_use;
ce736788 5128 u32 tx_cnt = priv->plat->tx_queues_to_use;
54139cf3
JP		 5129 u32 queue;
5130
5131 for (queue = 0; queue < rx_cnt; queue++) {
5132 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
5133
5134 rx_q->cur_rx = 0;
5135 rx_q->dirty_rx = 0;
5136 }
5137
ce736788
JP		 5138 for (queue = 0; queue < tx_cnt; queue++) {
5139 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
5140
5141 tx_q->cur_tx = 0;
5142 tx_q->dirty_tx = 0;
8d212a9e 5143 tx_q->mss = 0;
ce736788 5144 }
54139cf3
JP		 5145}
5146
732fdf0e
GC		 5147/**
5148 * stmmac_resume - resume callback
f4e7bd81 5149 * @dev: device pointer
732fdf0e
GC		 5150 * Description: when resume this function is invoked to setup the DMA and CORE
5151 * in a usable state.
5152 */
f4e7bd81 5153int stmmac_resume(struct device *dev)
47dd7a54 5154{
f4e7bd81 5155 struct net_device *ndev = dev_get_drvdata(dev);
874bd42d 5156 struct stmmac_priv *priv = netdev_priv(ndev);
b9663b7c 5157 int ret;
47dd7a54 5158
874bd42d 5159 if (!netif_running(ndev))
47dd7a54
GC		 5160 return 0;
5161
47dd7a54
GC		 5162 /* Power Down bit, into the PM register, is cleared
5163 * automatically as soon as a magic packet or a Wake-up frame
5164 * is received. Anyway, it's better to manually clear
5165 * this bit because it can generate problems while resuming
ceb69499
GC		 5166 * from another devices (e.g. serial console).
5167 */
623997fb 5168 if (device_may_wakeup(priv->device)) {
29555fa3 5169 mutex_lock(&priv->lock);
c10d4c82 5170 stmmac_pmt(priv, priv->hw, 0);
29555fa3 5171 mutex_unlock(&priv->lock);
89f7f2cf 5172 priv->irq_wake = 0;
623997fb 5173 } else {
db88f10a 5174 pinctrl_pm_select_default_state(priv->device);
8d45e42b 5175 /* enable the clk previously disabled */
e497c20e
BH		 5176 clk_prepare_enable(priv->plat->stmmac_clk);
5177 clk_prepare_enable(priv->plat->pclk);
5178 if (priv->plat->clk_ptp_ref)
5179 clk_prepare_enable(priv->plat->clk_ptp_ref);
623997fb
SK		 5180 /* reset the phy so that it's ready */
5181 if (priv->mii)
5182 stmmac_mdio_reset(priv->mii);
5183 }
47dd7a54 5184
b9663b7c
VW		 5185 if (priv->plat->serdes_powerup) {
5186 ret = priv->plat->serdes_powerup(ndev,
5187 priv->plat->bsp_priv);
5188
5189 if (ret < 0)
5190 return ret;
5191 }
5192
874bd42d 5193 netif_device_attach(ndev);
47dd7a54 5194
29555fa3 5195 mutex_lock(&priv->lock);
f55d84b0 5196
54139cf3
JP		 5197 stmmac_reset_queues_param(priv);
5198
ae79a639
GC		 5199 stmmac_clear_descriptors(priv);
5200
fe131929 5201 stmmac_hw_setup(ndev, false);
d429b66e 5202 stmmac_init_coalesce(priv);
ac316c78 5203 stmmac_set_rx_mode(ndev);
47dd7a54 5204
ed64639b
WVK		 5205 stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw);
5206
c22a3f48 5207 stmmac_enable_all_queues(priv);
47dd7a54 5208
c22a3f48 5209 stmmac_start_all_queues(priv);
47dd7a54 5210
19e13cb2 5211 mutex_unlock(&priv->lock);
102463b1 5212
3e2bf04f
JA		 5213 if (!device_may_wakeup(priv->device)) {
5214 rtnl_lock();
5215 phylink_start(priv->phylink);
5216 rtnl_unlock();
5217 }
5218
5219 phylink_mac_change(priv->phylink, true);
19e13cb2 5220
47dd7a54
GC		 5221 return 0;
5222}
b2e2f0c7 5223EXPORT_SYMBOL_GPL(stmmac_resume);
ba27ec66 5224
47dd7a54
GC		 5225#ifndef MODULE
5226static int __init stmmac_cmdline_opt(char *str)
5227{
5228 char *opt;
5229
5230 if (!str || !*str)
5231 return -EINVAL;
5232 while ((opt = strsep(&str, ",")) != NULL) {
f3240e28 5233 if (!strncmp(opt, "debug:", 6)) {
ea2ab871 5234 if (kstrtoint(opt + 6, 0, &debug))
f3240e28
GC		 5235 goto err;
5236 } else if (!strncmp(opt, "phyaddr:", 8)) {
ea2ab871 5237 if (kstrtoint(opt + 8, 0, &phyaddr))
f3240e28 5238 goto err;
f3240e28 5239 } else if (!strncmp(opt, "buf_sz:", 7)) {
ea2ab871 5240 if (kstrtoint(opt + 7, 0, &buf_sz))
f3240e28
GC		 5241 goto err;
5242 } else if (!strncmp(opt, "tc:", 3)) {
ea2ab871 5243 if (kstrtoint(opt + 3, 0, &tc))
f3240e28
GC		 5244 goto err;
5245 } else if (!strncmp(opt, "watchdog:", 9)) {
ea2ab871 5246 if (kstrtoint(opt + 9, 0, &watchdog))
f3240e28
GC		 5247 goto err;
5248 } else if (!strncmp(opt, "flow_ctrl:", 10)) {
ea2ab871 5249 if (kstrtoint(opt + 10, 0, &flow_ctrl))
f3240e28
GC		 5250 goto err;
5251 } else if (!strncmp(opt, "pause:", 6)) {
ea2ab871 5252 if (kstrtoint(opt + 6, 0, &pause))
f3240e28 5253 goto err;
506f669c 5254 } else if (!strncmp(opt, "eee_timer:", 10)) {
d765955d
GC		 5255 if (kstrtoint(opt + 10, 0, &eee_timer))
5256 goto err;
4a7d666a
GC		 5257 } else if (!strncmp(opt, "chain_mode:", 11)) {
5258 if (kstrtoint(opt + 11, 0, &chain_mode))
5259 goto err;
f3240e28 5260 }
47dd7a54
GC		 5261 }
5262 return 0;
f3240e28
GC		 5263
5264err:
5265 pr_err("%s: ERROR broken module parameter conversion", __func__);
5266 return -EINVAL;
47dd7a54
GC		 5267}
5268
5269__setup("stmmaceth=", stmmac_cmdline_opt);
ceb69499 5270#endif /* MODULE */
6fc0d0f2 5271
466c5ac8
MO		 5272static int __init stmmac_init(void)
5273{
5274#ifdef CONFIG_DEBUG_FS
5275 /* Create debugfs main directory if it doesn't exist yet */
8d72ab11 5276 if (!stmmac_fs_dir)
466c5ac8 5277 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
474a31e1 5278 register_netdevice_notifier(&stmmac_notifier);
466c5ac8
MO		 5279#endif
5280
5281 return 0;
5282}
5283
5284static void __exit stmmac_exit(void)
5285{
5286#ifdef CONFIG_DEBUG_FS
474a31e1 5287 unregister_netdevice_notifier(&stmmac_notifier);
466c5ac8
MO		 5288 debugfs_remove_recursive(stmmac_fs_dir);
5289#endif
5290}
5291
5292module_init(stmmac_init)
5293module_exit(stmmac_exit)
5294
6fc0d0f2
GC		 5295MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
5296MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
5297MODULE_LICENSE("GPL");
A mirror of Linus' kernel repository